862 lines
31 KiB
Python
862 lines
31 KiB
Python
"""runner.py 单元测试（算法保真 #13）。
|
||
|
||
覆盖 13a-13e 五个子任务，测试 Runner 骨架、三级嵌套、gate/accept/reject/probation、
|
||
慢更新十步序、deliver_best + early stop。大部分测试用纯函数或 mock 构造避免真实推理。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
from dataclasses import dataclass, field
|
||
from pathlib import Path # noqa: TC003 — 运行时 tmp_path 标注使用
|
||
from unittest.mock import MagicMock, patch
|
||
|
||
import pytest
|
||
|
||
from app.harness.runner import (
|
||
Runner,
|
||
_apply_batch_correctness,
|
||
_batch_from_ids,
|
||
_build_comparison_pairs,
|
||
_compute_total_steps,
|
||
_fallback_summary,
|
||
_filter_untrainable_types,
|
||
_format_applied_edits,
|
||
_guard_infra_failures,
|
||
_outcome_to_quadrant_pairs,
|
||
_should_early_stop,
|
||
_snapshot_current_skills,
|
||
_TrainState,
|
||
_write_skip_report,
|
||
resume_plan,
|
||
)
|
||
from app.harness.validate import Probation, ValidationOutcome
|
||
from core.evolution import RejectedEdit
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 测试辅助
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class _FakeInferenceResult:
|
||
"""InferenceResult 替身。"""
|
||
|
||
run_id: str = "test_run"
|
||
accuracy: float = 0.5
|
||
total: int = 10
|
||
correct: int = 5
|
||
per_task_type: dict = field(default_factory=dict)
|
||
steps_mean: float = 3.0
|
||
token_usage: dict = field(default_factory=lambda: {"prompt_tokens": 0, "completion_tokens": 0})
|
||
stop_reason_counts: dict = field(default_factory=lambda: {"finished": 10})
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class _FakeQuestion:
|
||
"""GeneratedQuestion 替身（含 pair 契约字段，供 build_units 聚合）。"""
|
||
|
||
question_id: str
|
||
video_id: str = "v1"
|
||
task_type: str = "Action Reasoning"
|
||
question: str = "问题"
|
||
options: tuple = ("A", "B", "C", "D")
|
||
answer: str = "A"
|
||
source_nodes: tuple = ()
|
||
difficulty: str = "medium"
|
||
pair_id: str | None = None
|
||
question_role: str = "single"
|
||
unit_id: str = ""
|
||
flip_axis: str | None = None
|
||
|
||
def __post_init__(self) -> None:
|
||
"""缺省 unit_id 回填为 pair_id 或 question_id，对齐真实 GeneratedQuestion。"""
|
||
if not self.unit_id:
|
||
object.__setattr__(self, "unit_id", self.pair_id or self.question_id)
|
||
|
||
|
||
def _fake_pair(pair_id: str, task_type: str, video_id: str = "v1") -> list[_FakeQuestion]:
|
||
"""构造合法孪生对（original + mirror），共享 pair_id/video_id/task_type/flip_axis。"""
|
||
return [
|
||
_FakeQuestion(
|
||
question_id=f"{pair_id}-o",
|
||
video_id=video_id,
|
||
task_type=task_type,
|
||
pair_id=pair_id,
|
||
question_role="pair_original",
|
||
),
|
||
_FakeQuestion(
|
||
question_id=f"{pair_id}-m",
|
||
video_id=video_id,
|
||
task_type=task_type,
|
||
pair_id=pair_id,
|
||
question_role="pair_mirror",
|
||
),
|
||
]
|
||
|
||
|
||
@dataclass
|
||
class _FakePools:
|
||
"""Pools 替身。"""
|
||
|
||
diagnosis: list = field(default_factory=list)
|
||
validation: list = field(default_factory=list)
|
||
test: list = field(default_factory=list)
|
||
baseline_run_id: str = "baseline_run"
|
||
baseline_val_accuracy: float = 0.5
|
||
correctness: dict = field(default_factory=dict)
|
||
|
||
|
||
# =========================================================================
|
||
# 13a: Runner 骨架 + 纯函数
|
||
# =========================================================================
|
||
|
||
|
||
class TestResumePlan:
|
||
"""resume_plan 纯函数。"""
|
||
|
||
def test_epoch_done_advances_epoch(self) -> None:
|
||
"""epoch_done 阶段：下一 epoch 从头开始。"""
|
||
plan = resume_plan(epoch=3, phase="epoch_done", step_completed=5)
|
||
assert plan["first_epoch"] == 4
|
||
assert plan["resume_epoch"] is None
|
||
assert plan["resume_step_from"] == 0
|
||
|
||
def test_in_epoch_resumes_same_epoch(self) -> None:
|
||
"""in_epoch 阶段：从同 epoch 的下一个 step 续跑。"""
|
||
plan = resume_plan(epoch=2, phase="in_epoch", step_completed=3)
|
||
assert plan["first_epoch"] == 2
|
||
assert plan["resume_epoch"] == 2
|
||
assert plan["resume_step_from"] == 4
|
||
|
||
def test_in_epoch_step_zero(self) -> None:
|
||
"""in_epoch step_completed=0：从 step 1 续跑。"""
|
||
plan = resume_plan(epoch=1, phase="in_epoch", step_completed=0)
|
||
assert plan["resume_step_from"] == 1
|
||
|
||
|
||
class TestGuardInfraFailures:
|
||
"""_guard_infra_failures 基础设施护栏。"""
|
||
|
||
def test_low_error_rate_passes(self) -> None:
|
||
"""error 率 <= 10% 不抛异常。"""
|
||
result = _FakeInferenceResult(
|
||
total=100,
|
||
stop_reason_counts={"finished": 95, "error": 5},
|
||
)
|
||
_guard_infra_failures(result, context="test") # 不应抛异常
|
||
|
||
def test_high_error_rate_raises(self) -> None:
|
||
"""error 率 > 10% 抛 RuntimeError。"""
|
||
result = _FakeInferenceResult(
|
||
total=10,
|
||
stop_reason_counts={"finished": 8, "error": 2},
|
||
)
|
||
with pytest.raises(RuntimeError, match="基础设施失败率过高"):
|
||
_guard_infra_failures(result, context="test")
|
||
|
||
def test_zero_total_does_not_crash(self) -> None:
|
||
"""total=0 时不除零崩溃。"""
|
||
result = _FakeInferenceResult(total=0, stop_reason_counts={})
|
||
_guard_infra_failures(result, context="test") # 不应抛异常
|
||
|
||
def test_no_error_key_passes(self) -> None:
|
||
"""stop_reason_counts 无 error 键时正常通过。"""
|
||
result = _FakeInferenceResult(
|
||
total=10,
|
||
stop_reason_counts={"finished": 10},
|
||
)
|
||
_guard_infra_failures(result, context="test")
|
||
|
||
|
||
# =========================================================================
|
||
# 13b: _apply_batch_correctness + _compute_total_steps
|
||
# =========================================================================
|
||
|
||
|
||
class TestApplyBatchCorrectness:
|
||
"""_apply_batch_correctness rollout 完整性护栏。"""
|
||
|
||
def test_complete_batch_updates_correctness(self) -> None:
|
||
"""完整 rollout 正常更新 correctness。"""
|
||
batch = [_FakeQuestion(question_id="q1"), _FakeQuestion(question_id="q2")]
|
||
correctness: dict[str, bool] = {}
|
||
|
||
# mock HarnessLog
|
||
mock_log = MagicMock()
|
||
mock_log.query.return_value = [
|
||
{"question_id": "q1", "prediction": "A", "answer": "A", "steps_json": "[]"},
|
||
{"question_id": "q2", "prediction": "B", "answer": "A", "steps_json": "[]"},
|
||
]
|
||
|
||
with patch("app.harness.validate._load_run_rows") as mock_load:
|
||
mock_load.return_value = {
|
||
"q1": {"prediction": "A", "answer": "A", "_correct": True, "steps": []},
|
||
"q2": {"prediction": "B", "answer": "A", "_correct": False, "steps": []},
|
||
}
|
||
_apply_batch_correctness(correctness, mock_log, "run_1", batch)
|
||
|
||
assert correctness["q1"] is True
|
||
assert correctness["q2"] is False
|
||
|
||
def test_missing_prediction_raises(self) -> None:
|
||
"""rollout 缺预测行时抛 RuntimeError。"""
|
||
batch = [_FakeQuestion(question_id="q1"), _FakeQuestion(question_id="q2")]
|
||
correctness: dict[str, bool] = {}
|
||
|
||
mock_log = MagicMock()
|
||
with patch("app.harness.validate._load_run_rows") as mock_load:
|
||
mock_load.return_value = {
|
||
"q1": {"prediction": "A", "answer": "A", "_correct": True, "steps": []},
|
||
# q2 缺失
|
||
}
|
||
with pytest.raises(RuntimeError, match="rollout 不完整"):
|
||
_apply_batch_correctness(correctness, mock_log, "run_1", batch)
|
||
|
||
|
||
class TestComputeTotalSteps:
|
||
"""_compute_total_steps 退火地平线。"""
|
||
|
||
def test_basic_calculation(self) -> None:
|
||
"""基本退火地平线计算。"""
|
||
questions = [
|
||
_FakeQuestion(question_id=f"q{i}", task_type="Action Reasoning") for i in range(20)
|
||
]
|
||
# 全错题
|
||
correctness = {q.question_id: False for q in questions}
|
||
|
||
config = MagicMock()
|
||
config.batch_size = 5
|
||
config.min_class_per_batch = 1
|
||
config.batch_correct_ratio = 0.0
|
||
config.epochs = 3
|
||
|
||
pools = _FakePools(diagnosis=questions, correctness=correctness)
|
||
total = _compute_total_steps(pools, correctness, config)
|
||
# 20 题 / batch_size 5 = 4 步/epoch * 3 epochs = 12
|
||
assert total == 12
|
||
|
||
|
||
# =========================================================================
|
||
# 13b: _batch_from_ids
|
||
# =========================================================================
|
||
|
||
|
||
class TestBatchFromIds:
|
||
"""_batch_from_ids 按 ID 重建 batch。"""
|
||
|
||
def test_preserves_order(self) -> None:
|
||
"""按 ids 顺序取出，保持原 batch 划分。"""
|
||
q1 = _FakeQuestion(question_id="q1")
|
||
q2 = _FakeQuestion(question_id="q2")
|
||
q3 = _FakeQuestion(question_id="q3")
|
||
pools = _FakePools(diagnosis=[q1, q2, q3])
|
||
|
||
batch = _batch_from_ids(pools, ["q3", "q1"])
|
||
assert [q.question_id for q in batch] == ["q3", "q1"]
|
||
|
||
|
||
# =========================================================================
|
||
# 13b: _snapshot_current_skills
|
||
# =========================================================================
|
||
|
||
|
||
class TestSnapshotCurrentSkills:
|
||
"""_snapshot_current_skills 快照 skill 文件。"""
|
||
|
||
def test_snapshots_md_files(self, tmp_path: Path) -> None:
|
||
"""只快照 .md 文件。"""
|
||
(tmp_path / "action-reasoning.md").write_text("skill content 1")
|
||
(tmp_path / "temporal.md").write_text("skill content 2")
|
||
(tmp_path / "meta.json").write_text("{}")
|
||
|
||
snapshot = _snapshot_current_skills(tmp_path)
|
||
assert "action-reasoning.md" in snapshot
|
||
assert "temporal.md" in snapshot
|
||
assert "meta.json" not in snapshot
|
||
assert snapshot["action-reasoning.md"] == "skill content 1"
|
||
|
||
|
||
# =========================================================================
|
||
# 13c: _outcome_to_quadrant_pairs
|
||
# =========================================================================
|
||
|
||
|
||
class TestOutcomeToQuadrantPairs:
|
||
"""_outcome_to_quadrant_pairs 四象限拍平。"""
|
||
|
||
def test_all_quadrants(self) -> None:
|
||
"""四象限各有一个 qid 时生成 4 条 pair。"""
|
||
outcome = ValidationOutcome(
|
||
action="accept_confirmed",
|
||
accepted=True,
|
||
stop_reason="confirmed",
|
||
e_value=10.0,
|
||
w=3,
|
||
l=0,
|
||
n_used=10,
|
||
delta_hat=0.3,
|
||
delta_shrunk=0.2,
|
||
baseline_acc=0.7,
|
||
candidate_acc=0.9,
|
||
improvements=["q1"],
|
||
regressions=["q2"],
|
||
persistent_fails=["q3"],
|
||
stable_successes=["q4"],
|
||
)
|
||
pairs = _outcome_to_quadrant_pairs("Action Reasoning", outcome)
|
||
assert len(pairs) == 4
|
||
by_qid = {p["question_id"]: p for p in pairs}
|
||
assert by_qid["q1"]["category"] == "improved"
|
||
assert by_qid["q1"]["prev_correct"] is False
|
||
assert by_qid["q1"]["curr_correct"] is True
|
||
assert by_qid["q2"]["category"] == "regressed"
|
||
assert by_qid["q3"]["category"] == "persistent_fail"
|
||
assert by_qid["q4"]["category"] == "stable_success"
|
||
|
||
def test_empty_outcome(self) -> None:
|
||
"""四象限全空时返回空列表。"""
|
||
outcome = ValidationOutcome(
|
||
action="reject",
|
||
accepted=False,
|
||
stop_reason="directional",
|
||
e_value=0.5,
|
||
w=0,
|
||
l=2,
|
||
n_used=5,
|
||
delta_hat=-0.1,
|
||
delta_shrunk=-0.05,
|
||
baseline_acc=0.8,
|
||
candidate_acc=0.6,
|
||
)
|
||
assert _outcome_to_quadrant_pairs("Any", outcome) == []
|
||
|
||
|
||
# =========================================================================
|
||
# 13c: _build_comparison_pairs
|
||
# =========================================================================
|
||
|
||
|
||
class TestBuildComparisonPairs:
|
||
"""_build_comparison_pairs momentum 纵向对比对。"""
|
||
|
||
def test_builds_pairs(self) -> None:
|
||
"""正确构造对比对。"""
|
||
sampled = [_FakeQuestion(question_id="q1", question="问题1")]
|
||
prev_rows = {
|
||
"q1": {"prediction": "A", "_correct": True},
|
||
}
|
||
curr_rows = {
|
||
"q1": {"prediction": "B", "_correct": False},
|
||
}
|
||
pairs = _build_comparison_pairs(sampled, prev_rows, curr_rows)
|
||
assert len(pairs) == 1
|
||
assert pairs[0]["question"] == "问题1"
|
||
assert pairs[0]["prev_prediction"] == "A"
|
||
assert pairs[0]["curr_prediction"] == "B"
|
||
assert pairs[0]["correct_prev"] is True
|
||
assert pairs[0]["correct_curr"] is False
|
||
|
||
def test_missing_rows_use_defaults(self) -> None:
|
||
"""缺失行时使用默认值。"""
|
||
sampled = [_FakeQuestion(question_id="q1")]
|
||
pairs = _build_comparison_pairs(sampled, {}, {})
|
||
assert pairs[0]["prev_prediction"] == ""
|
||
assert pairs[0]["correct_prev"] is False
|
||
|
||
|
||
# =========================================================================
|
||
# 13e: _should_early_stop
|
||
# =========================================================================
|
||
|
||
|
||
def _write_manifest_with_best(tmp_path: Path, best_epoch: int) -> None:
|
||
"""写含 best.epoch 的 manifest，供 _should_early_stop 读 read_best。"""
|
||
manifest = {
|
||
"name": "test",
|
||
"store": ".",
|
||
"current": {"videos": "v", "questions": "q", "skills": "s/v1", "prompts": "p/v1"},
|
||
"best": {"epoch": best_epoch, "val_acc": 0.5},
|
||
"history": [],
|
||
}
|
||
(tmp_path / "manifest.json").write_text(json.dumps(manifest))
|
||
|
||
|
||
class TestShouldEarlyStop:
|
||
"""_should_early_stop epoch 粒度 early stop（patience 以 epoch 计）。"""
|
||
|
||
def test_improved_this_epoch_resets(self, tmp_path: Path) -> None:
|
||
"""本 epoch best 刷新时重置计数器。"""
|
||
_write_manifest_with_best(tmp_path, best_epoch=2)
|
||
|
||
state = MagicMock()
|
||
state.epochs_since_best_improved = 3
|
||
|
||
result = _should_early_stop(tmp_path, epoch=2, state=state, patience=2)
|
||
assert result is False
|
||
assert state.epochs_since_best_improved == 0
|
||
|
||
def test_below_patience_continues(self, tmp_path: Path) -> None:
|
||
"""未达 patience 个 epoch 无刷新时继续。"""
|
||
_write_manifest_with_best(tmp_path, best_epoch=1)
|
||
|
||
state = MagicMock()
|
||
state.epochs_since_best_improved = 0
|
||
|
||
result = _should_early_stop(tmp_path, epoch=2, state=state, patience=3)
|
||
assert result is False
|
||
assert state.epochs_since_best_improved == 1
|
||
|
||
def test_early_stop_counts_epochs_not_steps(self, tmp_path: Path) -> None:
|
||
"""patience=2 表示连续 2 个 epoch 无 best 刷新才停（不是步数）。"""
|
||
_write_manifest_with_best(tmp_path, best_epoch=1)
|
||
|
||
state = MagicMock()
|
||
state.epochs_since_best_improved = 0
|
||
|
||
# epoch 2 无刷新 → 1 → 不停
|
||
assert _should_early_stop(tmp_path, epoch=2, state=state, patience=2) is False
|
||
assert state.epochs_since_best_improved == 1
|
||
# epoch 3 无刷新 → 2 → 停
|
||
assert _should_early_stop(tmp_path, epoch=3, state=state, patience=2) is True
|
||
assert state.epochs_since_best_improved == 2
|
||
|
||
|
||
class TestFilterUntrainableTypes:
|
||
"""_filter_untrainable_types 可训练性预检纯函数。"""
|
||
|
||
def test_untrainable_types_filtered_before_gate(self) -> None:
|
||
"""val<eval_min_per_class 或 非test单元<trainable_min_units 的题型被剔除。"""
|
||
# 题型 A：diag=5 + val=5 → units=10、val=5，可训
|
||
diag = [_FakeQuestion(question_id=f"A-d{i}", task_type="A") for i in range(5)]
|
||
val = [_FakeQuestion(question_id=f"A-v{i}", task_type="A") for i in range(5)]
|
||
# 题型 B：val=0（<eval_min_per_class）不可训
|
||
diag += [_FakeQuestion(question_id=f"B-d{i}", task_type="B") for i in range(2)]
|
||
pools = _FakePools(diagnosis=diag, validation=val, test=[])
|
||
|
||
new_pools, new_types = _filter_untrainable_types(
|
||
pools,
|
||
task_types=["A", "B"],
|
||
eval_min_per_class=2,
|
||
trainable_min_units=8,
|
||
)
|
||
|
||
assert {q.task_type for q in new_pools.diagnosis} == {"A"}
|
||
assert {q.task_type for q in new_pools.validation} == {"A"}
|
||
assert new_types == ["A"]
|
||
|
||
def test_units_below_threshold_filtered(self) -> None:
|
||
"""val 达标但 diag+val 单元数 < trainable_min_units 的题型被剔除（保留另一可训题型）。"""
|
||
# 可训题型 A：diag=8 + val=2 → units=10、val=2，保留
|
||
keep_diag = [_FakeQuestion(question_id=f"A-d{i}", task_type="A") for i in range(8)]
|
||
keep_val = [_FakeQuestion(question_id=f"A-v{i}", task_type="A") for i in range(2)]
|
||
# 题型 C：val=2（>=2）但 units=2+1=3 < 8 → 剔除
|
||
val = keep_val + [_FakeQuestion(question_id=f"C-v{i}", task_type="C") for i in range(2)]
|
||
diag = keep_diag + [_FakeQuestion(question_id="C-d0", task_type="C")]
|
||
pools = _FakePools(diagnosis=diag, validation=val, test=[])
|
||
|
||
new_pools, new_types = _filter_untrainable_types(
|
||
pools, task_types=None, eval_min_per_class=2, trainable_min_units=8
|
||
)
|
||
|
||
assert {q.task_type for q in new_pools.diagnosis} == {"A"}
|
||
assert {q.task_type for q in new_pools.validation} == {"A"}
|
||
assert new_types == ["A"]
|
||
|
||
def test_ar_pair_counted_as_units_not_questions(self) -> None:
|
||
"""AR pair 按单元折叠计数：题目数达标但单元数不足的题型仍被剔除。"""
|
||
# 可训题型 A：diag=8 + val=2 single → units=10，保留
|
||
keep_diag = [_FakeQuestion(question_id=f"A-d{i}", task_type="A") for i in range(8)]
|
||
keep_val = [_FakeQuestion(question_id=f"A-v{i}", task_type="A") for i in range(2)]
|
||
# 题型 P：diag 3 对（6 题=3 单元）+ val 2 对（4 题=2 单元）→ 单元数=5<8，
|
||
# 但题目数=10>=8。按单元计数须剔除（按题目计数会误通过）。
|
||
pair_diag: list[_FakeQuestion] = []
|
||
for i in range(3):
|
||
pair_diag.extend(_fake_pair(f"P-d{i}", "P"))
|
||
pair_val: list[_FakeQuestion] = []
|
||
for i in range(2):
|
||
pair_val.extend(_fake_pair(f"P-v{i}", "P"))
|
||
pools = _FakePools(
|
||
diagnosis=keep_diag + pair_diag,
|
||
validation=keep_val + pair_val,
|
||
test=[],
|
||
)
|
||
|
||
new_pools, new_types = _filter_untrainable_types(
|
||
pools, task_types=None, eval_min_per_class=2, trainable_min_units=8
|
||
)
|
||
|
||
assert "P" not in {q.task_type for q in new_pools.diagnosis}
|
||
assert "P" not in new_types
|
||
assert "A" in new_types
|
||
|
||
def test_all_filtered_raises(self) -> None:
|
||
"""所有题型都被剔除时 fail-fast：raise RuntimeError 并列出剔除原因。"""
|
||
diag = [_FakeQuestion(question_id="B-d0", task_type="B")] # val=0 < 2
|
||
pools = _FakePools(diagnosis=diag, validation=[], test=[])
|
||
|
||
with pytest.raises(RuntimeError, match="可训练"):
|
||
_filter_untrainable_types(
|
||
pools, task_types=None, eval_min_per_class=2, trainable_min_units=8
|
||
)
|
||
|
||
def test_test_pool_untouched(self) -> None:
|
||
"""test 池不参与过滤（继续报告全题型准确率）。"""
|
||
val = [_FakeQuestion(question_id=f"A-v{i}", task_type="A") for i in range(8)]
|
||
diag = [_FakeQuestion(question_id=f"A-d{i}", task_type="A") for i in range(8)]
|
||
test = [_FakeQuestion(question_id="B-t0", task_type="B")]
|
||
pools = _FakePools(diagnosis=diag, validation=val, test=test)
|
||
|
||
new_pools, _ = _filter_untrainable_types(
|
||
pools, task_types=None, eval_min_per_class=2, trainable_min_units=8
|
||
)
|
||
|
||
assert new_pools.test == test
|
||
|
||
|
||
# =========================================================================
|
||
# 13c: Probation 数据结构测试
|
||
# =========================================================================
|
||
|
||
|
||
class TestProbation:
|
||
"""Probation 数据结构。"""
|
||
|
||
def test_probation_fields(self) -> None:
|
||
"""Probation 必须具备全部字段。"""
|
||
p = Probation(
|
||
task_type="Action Reasoning",
|
||
anchor_skills_version="v1",
|
||
target_file="action-reasoning.md",
|
||
correctness_snapshot={"q1": True},
|
||
opened_step=5,
|
||
)
|
||
assert p.task_type == "Action Reasoning"
|
||
assert p.pending_edits == []
|
||
|
||
def test_pending_edits_append(self) -> None:
|
||
"""pending_edits 可追加 RejectedEdit。"""
|
||
p = Probation(
|
||
task_type="Action Reasoning",
|
||
anchor_skills_version="v1",
|
||
target_file="action-reasoning.md",
|
||
correctness_snapshot={},
|
||
opened_step=0,
|
||
)
|
||
edit = RejectedEdit(
|
||
target_file="action-reasoning.md",
|
||
target_type="skill",
|
||
change_summary="test",
|
||
delta=0.1,
|
||
source_version="v1",
|
||
epoch=0,
|
||
gate_w=3,
|
||
gate_l=1,
|
||
gate_e_value=2.5,
|
||
gate_delta_shrunk=0.05,
|
||
)
|
||
p.pending_edits.append(edit)
|
||
assert len(p.pending_edits) == 1
|
||
|
||
|
||
# =========================================================================
|
||
# 13c: RejectedSummary 黑名单防污染
|
||
# =========================================================================
|
||
|
||
|
||
class TestFormatAppliedEdits:
|
||
"""_format_applied_edits 只拼 applied 的 edit。"""
|
||
|
||
def test_only_applied_edits_in_summary(self) -> None:
|
||
"""只有 applied 状态的 edit 进入摘要。"""
|
||
record = MagicMock()
|
||
record.edits = [
|
||
{"op": "replace", "target": "section1"},
|
||
{"op": "insert", "content": "new_content"},
|
||
{"op": "delete", "target": "old_stuff"},
|
||
]
|
||
record.apply_report = [
|
||
{"status": "applied_exact"},
|
||
{"status": "skipped_not_found"},
|
||
{"status": "applied_fuzzy"},
|
||
]
|
||
|
||
summary = _format_applied_edits(record)
|
||
assert summary is not None
|
||
assert "section1" in summary
|
||
assert "old_stuff" in summary
|
||
assert "new_content" not in summary
|
||
|
||
def test_zero_applied_returns_info_message(self) -> None:
|
||
"""0 applied 返回信息性消息（非 None）。"""
|
||
record = MagicMock()
|
||
record.edits = [{"op": "replace", "target": "sec"}]
|
||
record.apply_report = [{"status": "skipped_not_found"}]
|
||
|
||
summary = _format_applied_edits(record)
|
||
assert summary is not None
|
||
assert "0 applied" in summary
|
||
|
||
def test_no_edits_returns_none(self) -> None:
|
||
"""无 edit 时返回 None。"""
|
||
record = MagicMock()
|
||
record.edits = []
|
||
|
||
assert _format_applied_edits(record) is None
|
||
|
||
def test_no_report_includes_all_edits(self) -> None:
|
||
"""无 apply_report 时包含所有 edit。"""
|
||
record = MagicMock()
|
||
record.edits = [{"op": "replace", "target": "foo"}]
|
||
record.apply_report = []
|
||
|
||
summary = _format_applied_edits(record)
|
||
assert summary is not None
|
||
assert "foo" in summary
|
||
|
||
|
||
class TestFallbackSummary:
|
||
"""_fallback_summary 兜底黑名单摘要。"""
|
||
|
||
def test_from_suggestions(self) -> None:
|
||
"""有 suggestions 时拼接 change 字段。"""
|
||
record = MagicMock()
|
||
record.suggestions = [{"change": "改 A"}, {"change": "改 B"}]
|
||
outcome = MagicMock()
|
||
outcome.delta_hat = -0.1
|
||
|
||
summary = _fallback_summary(record, outcome)
|
||
assert "改 A" in summary
|
||
assert "改 B" in summary
|
||
|
||
def test_no_suggestions_uses_delta(self) -> None:
|
||
"""无 suggestions 时使用 delta 信息。"""
|
||
record = MagicMock()
|
||
record.suggestions = []
|
||
outcome = MagicMock()
|
||
outcome.delta_hat = -0.15
|
||
|
||
summary = _fallback_summary(record, outcome)
|
||
assert "delta" in summary
|
||
assert "-0.15" in summary
|
||
|
||
|
||
class TestRejectedSummaryIntegration:
|
||
"""_rejected_summary_static 集成：两个子函数组合。"""
|
||
|
||
def test_static_delegates_to_format_applied(self) -> None:
|
||
"""有 applied edit 时 static 方法返回 _format_applied_edits 结果。"""
|
||
from app.harness.runner import Runner
|
||
|
||
record = MagicMock()
|
||
record.edits = [{"op": "replace", "target": "section1"}]
|
||
record.apply_report = [{"status": "applied_exact"}]
|
||
record.suggestions = []
|
||
outcome = MagicMock()
|
||
outcome.delta_hat = 0.1
|
||
|
||
summary = Runner._rejected_summary_static(record, outcome)
|
||
assert "section1" in summary
|
||
|
||
def test_static_falls_back_to_suggestions(self) -> None:
|
||
"""无 edit 时 static 方法使用 _fallback_summary。"""
|
||
from app.harness.runner import Runner
|
||
|
||
record = MagicMock()
|
||
record.edits = []
|
||
record.suggestions = [{"change": "尝试 X"}]
|
||
outcome = MagicMock()
|
||
outcome.delta_hat = -0.2
|
||
|
||
summary = Runner._rejected_summary_static(record, outcome)
|
||
assert "尝试 X" in summary
|
||
|
||
|
||
class TestWriteSkipReport:
|
||
"""_write_skip_report 辅助函数。"""
|
||
|
||
def test_writes_cooldown_report(self, tmp_path: Path) -> None:
|
||
"""cooldown 路径写 step_report JSON 文件。"""
|
||
(tmp_path / "analyses").mkdir()
|
||
_write_skip_report(
|
||
tmp_path,
|
||
epoch=1,
|
||
step=0,
|
||
global_step=5,
|
||
task_type="Action Reasoning",
|
||
action="cooldown",
|
||
baseline_acc=0.75,
|
||
budget=3,
|
||
)
|
||
report_path = tmp_path / "analyses" / "step_report_e1_s0_action-reasoning.json"
|
||
assert report_path.exists()
|
||
data = json.loads(report_path.read_text())
|
||
assert data["gate_action"] == "cooldown"
|
||
assert data["candidate_acc"] == 0.75
|
||
assert data["gate_w"] is None
|
||
|
||
def test_writes_skipped_report(self, tmp_path: Path) -> None:
|
||
"""skipped 路径写 step_report 并传递 rank_clip_triggered。"""
|
||
(tmp_path / "analyses").mkdir()
|
||
_write_skip_report(
|
||
tmp_path,
|
||
epoch=2,
|
||
step=1,
|
||
global_step=10,
|
||
task_type="Temporal Reasoning",
|
||
action="skipped",
|
||
baseline_acc=0.6,
|
||
budget=2,
|
||
rank_clip_triggered=True,
|
||
)
|
||
report_path = tmp_path / "analyses" / "step_report_e2_s1_temporal-reasoning.json"
|
||
assert report_path.exists()
|
||
data = json.loads(report_path.read_text())
|
||
assert data["gate_action"] == "skipped"
|
||
assert data["rank_clip_triggered"] is True
|
||
|
||
|
||
# =========================================================================
|
||
# 13a: _TrainState 基本构造
|
||
# =========================================================================
|
||
|
||
|
||
class TestTrainState:
|
||
"""_TrainState dataclass 基本构造与字段默认值。"""
|
||
|
||
def test_default_fields(self) -> None:
|
||
"""默认字段值正确。"""
|
||
state = _TrainState(
|
||
correctness={"q1": True},
|
||
gate_pools=MagicMock(),
|
||
baseline_cache=MagicMock(),
|
||
eval_prev_acc=0.5,
|
||
eval_prev_run_id="run1",
|
||
best_val_acc=0.5,
|
||
best_skills_version="v1",
|
||
best_prompts_version="v1",
|
||
)
|
||
assert state.global_step == 0
|
||
assert state.gate_epoch_observed is False
|
||
assert state.probations == {}
|
||
assert state.gate_cooldown == {}
|
||
assert state.rejected_buffer == {}
|
||
assert state.system_packs == []
|
||
assert state.tool_packs == []
|
||
assert state.changed_task_types_this_epoch == set()
|
||
assert state.epochs_since_best_improved == 0
|
||
|
||
|
||
# =========================================================================
|
||
# 13c: cooldown 递减测试
|
||
# =========================================================================
|
||
|
||
|
||
class TestCooldownDecrement:
|
||
"""gate_cooldown 每 step 递减、归零剔除。"""
|
||
|
||
def test_decrement_and_remove(self) -> None:
|
||
"""冷却值递减，归零剔除。"""
|
||
cooldown = {"type_a": 3, "type_b": 1, "type_c": 2}
|
||
# 模拟 _run_step 末尾的冷却递减
|
||
cooldown = {t: n - 1 for t, n in cooldown.items() if n - 1 > 0}
|
||
assert cooldown == {"type_a": 2, "type_c": 1}
|
||
|
||
cooldown = {t: n - 1 for t, n in cooldown.items() if n - 1 > 0}
|
||
assert cooldown == {"type_a": 1}
|
||
|
||
cooldown = {t: n - 1 for t, n in cooldown.items() if n - 1 > 0}
|
||
assert cooldown == {}
|
||
|
||
|
||
# =========================================================================
|
||
# factory 注入（Task 4）
|
||
# =========================================================================
|
||
|
||
|
||
class TestRunnerFactoryInjection:
|
||
"""Runner 构造时 tool_dispatch_factory / prompt_builder_factory 注入检查。"""
|
||
|
||
@staticmethod
|
||
def _base_config(tmp_path: Path, *, mode: str = "infer", **overrides):
|
||
"""构造 RunConfig，所有必填字段都给默认值。"""
|
||
from app.harness.config import RunConfig
|
||
|
||
defaults = {
|
||
"workspace_dir": tmp_path,
|
||
"store_dir": tmp_path,
|
||
"mode": mode,
|
||
"concurrency": 1,
|
||
"max_steps": 5,
|
||
"skill_mode": "none",
|
||
"n_samples": 0,
|
||
"questions": "benchmarks/Video-MME",
|
||
"skills_version": "v1",
|
||
"prompts_version": "v1",
|
||
"epochs": 1,
|
||
"diag_size": 10,
|
||
"diag_correct_ratio": 0.5,
|
||
"val_size": 24,
|
||
"val_correct_ratio": 0.5,
|
||
"edit_budget_start": 5,
|
||
"edit_budget_end": 2,
|
||
"batch_size": 5,
|
||
"min_class_per_batch": 2,
|
||
"eval_min_per_class": 2,
|
||
"trainable_min_units": 8,
|
||
"early_stop_patience": 3,
|
||
"test_size": 10,
|
||
"use_slow_momentum": False,
|
||
"gate_e_confirm": 20.0,
|
||
"gate_e_provisional": 3.0,
|
||
"gate_w_net_min": 2,
|
||
"gate_delta_min": 0.02,
|
||
"gate_lambda_dir": -0.642,
|
||
"gate_e_rollback": 10.0,
|
||
"gate_block": 8,
|
||
"gate_n_max": 40,
|
||
"gate_p_low": 0.05,
|
||
"gate_p_high": 0.95,
|
||
"gate_probe_quota": 0.2,
|
||
"gate_gamma_decay": 0.9,
|
||
"gate_cooldown_steps": 2,
|
||
"gate_guard_err": 0.10,
|
||
"skill_update_mode": "patch",
|
||
"appendix_consolidate_threshold": 6,
|
||
}
|
||
defaults.update(overrides)
|
||
return RunConfig(**defaults)
|
||
|
||
def test_infer_mode_missing_factory_raises(self, tmp_path: Path) -> None:
|
||
"""infer 模式缺少工厂时抛出 ValueError。"""
|
||
config = self._base_config(tmp_path, mode="infer")
|
||
with pytest.raises(ValueError, match="tool_dispatch_factory"):
|
||
Runner(
|
||
config,
|
||
llm=MagicMock(),
|
||
evolve_llm=MagicMock(),
|
||
vlm=MagicMock(),
|
||
telemetry=MagicMock(),
|
||
)
|
||
|
||
def test_diagnose_mode_allows_none_factory(self, tmp_path: Path) -> None:
|
||
"""diagnose 模式不需要工厂，允许 None。"""
|
||
ws = tmp_path / "ws"
|
||
ws.mkdir()
|
||
(ws / "manifest.json").write_text(
|
||
'{"name":"ws","created_at":"","store":"../store",'
|
||
'"current":{"videos":"v","questions":"q","skills":"s","prompts":"p"},'
|
||
'"history":[]}'
|
||
)
|
||
config = self._base_config(tmp_path, mode="diagnose", workspace_dir=ws, run_id="test_run")
|
||
# 不应抛出 ValueError
|
||
runner = Runner(
|
||
config,
|
||
llm=MagicMock(),
|
||
evolve_llm=MagicMock(),
|
||
vlm=MagicMock(),
|
||
telemetry=MagicMock(),
|
||
)
|
||
assert runner._tool_dispatch_factory is None
|
||
assert runner._prompt_builder_factory is None
|