iomgaa
2296134f73
Three-level nesting (epoch -> step -> per-skill), slow update 10-step sequence, checkpoint/resume, early stop, probation accept/reject/rollback. Key TRM4->TRM5 changes: - sync -> async (all inference/diagnosis/evolve/validate awaited) - LLMClient.from_env -> injected LLMProvider (DI via constructor) - Direct DB/file access -> module functions (workspace/store/log) - _TrainState as train() local, explicit param passing to helpers Module-level pure functions extracted for testability: resume_plan, _guard_infra_failures, _apply_batch_correctness, _compute_total_steps, _should_early_stop, _format_applied_edits, _fallback_summary, _write_skip_report, _outcome_to_quadrant_pairs, _build_comparison_pairs, _batch_from_ids, _snapshot_current_skills. Tests: 34 unit tests covering 13a-13e sub-tasks. Radon: all functions Grade B or better.
683 lines
24 KiB
Python
683 lines
24 KiB
Python
"""runner.py 单元测试（算法保真 #13）。
|
|
|
|
覆盖 13a-13e 五个子任务，测试 Runner 骨架、三级嵌套、gate/accept/reject/probation、
|
|
慢更新十步序、deliver_best + early stop。大部分测试用纯函数或 mock 构造避免真实推理。
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path # noqa: TC003 — 运行时 tmp_path 标注使用
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
from app.harness.runner import (
|
|
_apply_batch_correctness,
|
|
_batch_from_ids,
|
|
_build_comparison_pairs,
|
|
_compute_total_steps,
|
|
_fallback_summary,
|
|
_format_applied_edits,
|
|
_guard_infra_failures,
|
|
_outcome_to_quadrant_pairs,
|
|
_should_early_stop,
|
|
_snapshot_current_skills,
|
|
_TrainState,
|
|
_write_skip_report,
|
|
resume_plan,
|
|
)
|
|
from app.harness.validate import Probation, ValidationOutcome
|
|
from core.evolution import RejectedEdit
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 测试辅助
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class _FakeInferenceResult:
|
|
"""InferenceResult 替身。"""
|
|
|
|
run_id: str = "test_run"
|
|
accuracy: float = 0.5
|
|
total: int = 10
|
|
correct: int = 5
|
|
per_task_type: dict = field(default_factory=dict)
|
|
steps_mean: float = 3.0
|
|
token_usage: dict = field(default_factory=lambda: {"prompt_tokens": 0, "completion_tokens": 0})
|
|
stop_reason_counts: dict = field(default_factory=lambda: {"finished": 10})
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class _FakeQuestion:
|
|
"""GeneratedQuestion 替身。"""
|
|
|
|
question_id: str
|
|
video_id: str = "v1"
|
|
task_type: str = "Action Reasoning"
|
|
question: str = "问题"
|
|
options: tuple = ("A", "B", "C", "D")
|
|
answer: str = "A"
|
|
source_nodes: tuple = ()
|
|
difficulty: str = "medium"
|
|
|
|
|
|
@dataclass
|
|
class _FakePools:
|
|
"""Pools 替身。"""
|
|
|
|
diagnosis: list = field(default_factory=list)
|
|
validation: list = field(default_factory=list)
|
|
test: list = field(default_factory=list)
|
|
baseline_run_id: str = "baseline_run"
|
|
baseline_val_accuracy: float = 0.5
|
|
correctness: dict = field(default_factory=dict)
|
|
|
|
|
|
# =========================================================================
|
|
# 13a: Runner 骨架 + 纯函数
|
|
# =========================================================================
|
|
|
|
|
|
class TestResumePlan:
|
|
"""resume_plan 纯函数。"""
|
|
|
|
def test_epoch_done_advances_epoch(self) -> None:
|
|
"""epoch_done 阶段：下一 epoch 从头开始。"""
|
|
plan = resume_plan(epoch=3, phase="epoch_done", step_completed=5)
|
|
assert plan["first_epoch"] == 4
|
|
assert plan["resume_epoch"] is None
|
|
assert plan["resume_step_from"] == 0
|
|
|
|
def test_in_epoch_resumes_same_epoch(self) -> None:
|
|
"""in_epoch 阶段：从同 epoch 的下一个 step 续跑。"""
|
|
plan = resume_plan(epoch=2, phase="in_epoch", step_completed=3)
|
|
assert plan["first_epoch"] == 2
|
|
assert plan["resume_epoch"] == 2
|
|
assert plan["resume_step_from"] == 4
|
|
|
|
def test_in_epoch_step_zero(self) -> None:
|
|
"""in_epoch step_completed=0：从 step 1 续跑。"""
|
|
plan = resume_plan(epoch=1, phase="in_epoch", step_completed=0)
|
|
assert plan["resume_step_from"] == 1
|
|
|
|
|
|
class TestGuardInfraFailures:
|
|
"""_guard_infra_failures 基础设施护栏。"""
|
|
|
|
def test_low_error_rate_passes(self) -> None:
|
|
"""error 率 <= 10% 不抛异常。"""
|
|
result = _FakeInferenceResult(
|
|
total=100,
|
|
stop_reason_counts={"finished": 95, "error": 5},
|
|
)
|
|
_guard_infra_failures(result, context="test") # 不应抛异常
|
|
|
|
def test_high_error_rate_raises(self) -> None:
|
|
"""error 率 > 10% 抛 RuntimeError。"""
|
|
result = _FakeInferenceResult(
|
|
total=10,
|
|
stop_reason_counts={"finished": 8, "error": 2},
|
|
)
|
|
with pytest.raises(RuntimeError, match="基础设施失败率过高"):
|
|
_guard_infra_failures(result, context="test")
|
|
|
|
def test_zero_total_does_not_crash(self) -> None:
|
|
"""total=0 时不除零崩溃。"""
|
|
result = _FakeInferenceResult(total=0, stop_reason_counts={})
|
|
_guard_infra_failures(result, context="test") # 不应抛异常
|
|
|
|
def test_no_error_key_passes(self) -> None:
|
|
"""stop_reason_counts 无 error 键时正常通过。"""
|
|
result = _FakeInferenceResult(
|
|
total=10,
|
|
stop_reason_counts={"finished": 10},
|
|
)
|
|
_guard_infra_failures(result, context="test")
|
|
|
|
|
|
# =========================================================================
|
|
# 13b: _apply_batch_correctness + _compute_total_steps
|
|
# =========================================================================
|
|
|
|
|
|
class TestApplyBatchCorrectness:
|
|
"""_apply_batch_correctness rollout 完整性护栏。"""
|
|
|
|
def test_complete_batch_updates_correctness(self) -> None:
|
|
"""完整 rollout 正常更新 correctness。"""
|
|
batch = [_FakeQuestion(question_id="q1"), _FakeQuestion(question_id="q2")]
|
|
correctness: dict[str, bool] = {}
|
|
|
|
# mock HarnessLog
|
|
mock_log = MagicMock()
|
|
mock_log.query.return_value = [
|
|
{"question_id": "q1", "prediction": "A", "answer": "A", "steps_json": "[]"},
|
|
{"question_id": "q2", "prediction": "B", "answer": "A", "steps_json": "[]"},
|
|
]
|
|
|
|
with patch("app.harness.validate._load_run_rows") as mock_load:
|
|
mock_load.return_value = {
|
|
"q1": {"prediction": "A", "answer": "A", "_correct": True, "steps": []},
|
|
"q2": {"prediction": "B", "answer": "A", "_correct": False, "steps": []},
|
|
}
|
|
_apply_batch_correctness(correctness, mock_log, "run_1", batch)
|
|
|
|
assert correctness["q1"] is True
|
|
assert correctness["q2"] is False
|
|
|
|
def test_missing_prediction_raises(self) -> None:
|
|
"""rollout 缺预测行时抛 RuntimeError。"""
|
|
batch = [_FakeQuestion(question_id="q1"), _FakeQuestion(question_id="q2")]
|
|
correctness: dict[str, bool] = {}
|
|
|
|
mock_log = MagicMock()
|
|
with patch("app.harness.validate._load_run_rows") as mock_load:
|
|
mock_load.return_value = {
|
|
"q1": {"prediction": "A", "answer": "A", "_correct": True, "steps": []},
|
|
# q2 缺失
|
|
}
|
|
with pytest.raises(RuntimeError, match="rollout 不完整"):
|
|
_apply_batch_correctness(correctness, mock_log, "run_1", batch)
|
|
|
|
|
|
class TestComputeTotalSteps:
|
|
"""_compute_total_steps 退火地平线。"""
|
|
|
|
def test_basic_calculation(self) -> None:
|
|
"""基本退火地平线计算。"""
|
|
questions = [
|
|
_FakeQuestion(question_id=f"q{i}", task_type="Action Reasoning") for i in range(20)
|
|
]
|
|
# 全错题
|
|
correctness = {q.question_id: False for q in questions}
|
|
|
|
config = MagicMock()
|
|
config.batch_size = 5
|
|
config.min_class_per_batch = 1
|
|
config.batch_correct_ratio = 0.0
|
|
config.epochs = 3
|
|
|
|
pools = _FakePools(diagnosis=questions, correctness=correctness)
|
|
total = _compute_total_steps(pools, correctness, config)
|
|
# 20 题 / batch_size 5 = 4 步/epoch * 3 epochs = 12
|
|
assert total == 12
|
|
|
|
|
|
# =========================================================================
|
|
# 13b: _batch_from_ids
|
|
# =========================================================================
|
|
|
|
|
|
class TestBatchFromIds:
|
|
"""_batch_from_ids 按 ID 重建 batch。"""
|
|
|
|
def test_preserves_order(self) -> None:
|
|
"""按 ids 顺序取出，保持原 batch 划分。"""
|
|
q1 = _FakeQuestion(question_id="q1")
|
|
q2 = _FakeQuestion(question_id="q2")
|
|
q3 = _FakeQuestion(question_id="q3")
|
|
pools = _FakePools(diagnosis=[q1, q2, q3])
|
|
|
|
batch = _batch_from_ids(pools, ["q3", "q1"])
|
|
assert [q.question_id for q in batch] == ["q3", "q1"]
|
|
|
|
|
|
# =========================================================================
|
|
# 13b: _snapshot_current_skills
|
|
# =========================================================================
|
|
|
|
|
|
class TestSnapshotCurrentSkills:
|
|
"""_snapshot_current_skills 快照 skill 文件。"""
|
|
|
|
def test_snapshots_md_files(self, tmp_path: Path) -> None:
|
|
"""只快照 .md 文件。"""
|
|
(tmp_path / "action-reasoning.md").write_text("skill content 1")
|
|
(tmp_path / "temporal.md").write_text("skill content 2")
|
|
(tmp_path / "meta.json").write_text("{}")
|
|
|
|
snapshot = _snapshot_current_skills(tmp_path)
|
|
assert "action-reasoning.md" in snapshot
|
|
assert "temporal.md" in snapshot
|
|
assert "meta.json" not in snapshot
|
|
assert snapshot["action-reasoning.md"] == "skill content 1"
|
|
|
|
|
|
# =========================================================================
|
|
# 13c: _outcome_to_quadrant_pairs
|
|
# =========================================================================
|
|
|
|
|
|
class TestOutcomeToQuadrantPairs:
|
|
"""_outcome_to_quadrant_pairs 四象限拍平。"""
|
|
|
|
def test_all_quadrants(self) -> None:
|
|
"""四象限各有一个 qid 时生成 4 条 pair。"""
|
|
outcome = ValidationOutcome(
|
|
action="accept_confirmed",
|
|
accepted=True,
|
|
stop_reason="confirmed",
|
|
e_value=10.0,
|
|
w=3,
|
|
l=0,
|
|
n_used=10,
|
|
delta_hat=0.3,
|
|
delta_shrunk=0.2,
|
|
baseline_acc=0.7,
|
|
candidate_acc=0.9,
|
|
improvements=["q1"],
|
|
regressions=["q2"],
|
|
persistent_fails=["q3"],
|
|
stable_successes=["q4"],
|
|
)
|
|
pairs = _outcome_to_quadrant_pairs("Action Reasoning", outcome)
|
|
assert len(pairs) == 4
|
|
by_qid = {p["question_id"]: p for p in pairs}
|
|
assert by_qid["q1"]["category"] == "improved"
|
|
assert by_qid["q1"]["prev_correct"] is False
|
|
assert by_qid["q1"]["curr_correct"] is True
|
|
assert by_qid["q2"]["category"] == "regressed"
|
|
assert by_qid["q3"]["category"] == "persistent_fail"
|
|
assert by_qid["q4"]["category"] == "stable_success"
|
|
|
|
def test_empty_outcome(self) -> None:
|
|
"""四象限全空时返回空列表。"""
|
|
outcome = ValidationOutcome(
|
|
action="reject",
|
|
accepted=False,
|
|
stop_reason="directional",
|
|
e_value=0.5,
|
|
w=0,
|
|
l=2,
|
|
n_used=5,
|
|
delta_hat=-0.1,
|
|
delta_shrunk=-0.05,
|
|
baseline_acc=0.8,
|
|
candidate_acc=0.6,
|
|
)
|
|
assert _outcome_to_quadrant_pairs("Any", outcome) == []
|
|
|
|
|
|
# =========================================================================
|
|
# 13c: _build_comparison_pairs
|
|
# =========================================================================
|
|
|
|
|
|
class TestBuildComparisonPairs:
|
|
"""_build_comparison_pairs momentum 纵向对比对。"""
|
|
|
|
def test_builds_pairs(self) -> None:
|
|
"""正确构造对比对。"""
|
|
sampled = [_FakeQuestion(question_id="q1", question="问题1")]
|
|
prev_rows = {
|
|
"q1": {"prediction": "A", "_correct": True},
|
|
}
|
|
curr_rows = {
|
|
"q1": {"prediction": "B", "_correct": False},
|
|
}
|
|
pairs = _build_comparison_pairs(sampled, prev_rows, curr_rows)
|
|
assert len(pairs) == 1
|
|
assert pairs[0]["question"] == "问题1"
|
|
assert pairs[0]["prev_prediction"] == "A"
|
|
assert pairs[0]["curr_prediction"] == "B"
|
|
assert pairs[0]["correct_prev"] is True
|
|
assert pairs[0]["correct_curr"] is False
|
|
|
|
def test_missing_rows_use_defaults(self) -> None:
|
|
"""缺失行时使用默认值。"""
|
|
sampled = [_FakeQuestion(question_id="q1")]
|
|
pairs = _build_comparison_pairs(sampled, {}, {})
|
|
assert pairs[0]["prev_prediction"] == ""
|
|
assert pairs[0]["correct_prev"] is False
|
|
|
|
|
|
# =========================================================================
|
|
# 13e: _should_early_stop
|
|
# =========================================================================
|
|
|
|
|
|
class TestShouldEarlyStop:
|
|
"""_should_early_stop 步粒度 early stop。"""
|
|
|
|
def test_improved_this_epoch_resets(self, tmp_path: Path) -> None:
|
|
"""本 epoch best 刷新时重置计数器。"""
|
|
# 写 manifest + best
|
|
manifest = {
|
|
"name": "test",
|
|
"store": ".",
|
|
"current": {"videos": "v", "questions": "q", "skills": "s/v1", "prompts": "p/v1"},
|
|
"best": {"epoch": 2, "val_acc": 0.9},
|
|
"history": [],
|
|
}
|
|
(tmp_path / "manifest.json").write_text(json.dumps(manifest))
|
|
|
|
state = MagicMock()
|
|
state.steps_since_best_improved = 10
|
|
|
|
result = _should_early_stop(tmp_path, epoch=2, steps_this_epoch=5, state=state, patience=20)
|
|
assert result is False
|
|
assert state.steps_since_best_improved == 0
|
|
|
|
def test_no_improvement_accumulates(self, tmp_path: Path) -> None:
|
|
"""未刷新时累加步数。"""
|
|
manifest = {
|
|
"name": "test",
|
|
"store": ".",
|
|
"current": {"videos": "v", "questions": "q", "skills": "s/v1", "prompts": "p/v1"},
|
|
"best": {"epoch": 1, "val_acc": 0.5},
|
|
"history": [],
|
|
}
|
|
(tmp_path / "manifest.json").write_text(json.dumps(manifest))
|
|
|
|
state = MagicMock()
|
|
state.steps_since_best_improved = 15
|
|
|
|
result = _should_early_stop(tmp_path, epoch=3, steps_this_epoch=5, state=state, patience=20)
|
|
assert result is True # 15 + 5 = 20 >= 20
|
|
assert state.steps_since_best_improved == 20
|
|
|
|
def test_below_patience_continues(self, tmp_path: Path) -> None:
|
|
"""累计步数未达阈值时继续。"""
|
|
manifest = {
|
|
"name": "test",
|
|
"store": ".",
|
|
"current": {"videos": "v", "questions": "q", "skills": "s/v1", "prompts": "p/v1"},
|
|
"best": {"epoch": 1, "val_acc": 0.5},
|
|
"history": [],
|
|
}
|
|
(tmp_path / "manifest.json").write_text(json.dumps(manifest))
|
|
|
|
state = MagicMock()
|
|
state.steps_since_best_improved = 10
|
|
|
|
result = _should_early_stop(tmp_path, epoch=3, steps_this_epoch=5, state=state, patience=20)
|
|
assert result is False
|
|
assert state.steps_since_best_improved == 15
|
|
|
|
def test_step_granularity(self, tmp_path: Path) -> None:
|
|
"""步粒度而非 epoch 粒度。"""
|
|
manifest = {
|
|
"name": "test",
|
|
"store": ".",
|
|
"current": {"videos": "v", "questions": "q", "skills": "s/v1", "prompts": "p/v1"},
|
|
"best": {"epoch": 1, "val_acc": 0.5},
|
|
"history": [],
|
|
}
|
|
(tmp_path / "manifest.json").write_text(json.dumps(manifest))
|
|
|
|
state = MagicMock()
|
|
# 连续 3 个 epoch，每个 3 步
|
|
state.steps_since_best_improved = 0
|
|
for ep in range(2, 5):
|
|
stopped = _should_early_stop(
|
|
tmp_path, epoch=ep, steps_this_epoch=3, state=state, patience=10
|
|
)
|
|
if ep < 4:
|
|
assert stopped is False
|
|
else:
|
|
# 3+3+3=9 < 10 但第三轮后 9+3=12>=10 在 ep=5 触发
|
|
# 实际：ep=2 → 3, ep=3 → 6, ep=4 → 9
|
|
assert stopped is False
|
|
stopped = _should_early_stop(
|
|
tmp_path, epoch=5, steps_this_epoch=3, state=state, patience=10
|
|
)
|
|
assert stopped is True # 9+3=12>=10
|
|
|
|
|
|
# =========================================================================
|
|
# 13c: Probation 数据结构测试
|
|
# =========================================================================
|
|
|
|
|
|
class TestProbation:
|
|
"""Probation 数据结构。"""
|
|
|
|
def test_probation_fields(self) -> None:
|
|
"""Probation 必须具备全部字段。"""
|
|
p = Probation(
|
|
task_type="Action Reasoning",
|
|
anchor_skills_version="v1",
|
|
target_file="action-reasoning.md",
|
|
correctness_snapshot={"q1": True},
|
|
opened_step=5,
|
|
)
|
|
assert p.task_type == "Action Reasoning"
|
|
assert p.pending_edits == []
|
|
|
|
def test_pending_edits_append(self) -> None:
|
|
"""pending_edits 可追加 RejectedEdit。"""
|
|
p = Probation(
|
|
task_type="Action Reasoning",
|
|
anchor_skills_version="v1",
|
|
target_file="action-reasoning.md",
|
|
correctness_snapshot={},
|
|
opened_step=0,
|
|
)
|
|
edit = RejectedEdit(
|
|
target_file="action-reasoning.md",
|
|
target_type="skill",
|
|
change_summary="test",
|
|
delta=0.1,
|
|
source_version="v1",
|
|
epoch=0,
|
|
gate_w=3,
|
|
gate_l=1,
|
|
gate_e_value=2.5,
|
|
gate_delta_shrunk=0.05,
|
|
)
|
|
p.pending_edits.append(edit)
|
|
assert len(p.pending_edits) == 1
|
|
|
|
|
|
# =========================================================================
|
|
# 13c: RejectedSummary 黑名单防污染
|
|
# =========================================================================
|
|
|
|
|
|
class TestFormatAppliedEdits:
|
|
"""_format_applied_edits 只拼 applied 的 edit。"""
|
|
|
|
def test_only_applied_edits_in_summary(self) -> None:
|
|
"""只有 applied 状态的 edit 进入摘要。"""
|
|
record = MagicMock()
|
|
record.edits = [
|
|
{"op": "replace", "target": "section1"},
|
|
{"op": "insert", "content": "new_content"},
|
|
{"op": "delete", "target": "old_stuff"},
|
|
]
|
|
record.apply_report = [
|
|
{"status": "applied_exact"},
|
|
{"status": "skipped_not_found"},
|
|
{"status": "applied_fuzzy"},
|
|
]
|
|
|
|
summary = _format_applied_edits(record)
|
|
assert summary is not None
|
|
assert "section1" in summary
|
|
assert "old_stuff" in summary
|
|
assert "new_content" not in summary
|
|
|
|
def test_zero_applied_returns_info_message(self) -> None:
|
|
"""0 applied 返回信息性消息（非 None）。"""
|
|
record = MagicMock()
|
|
record.edits = [{"op": "replace", "target": "sec"}]
|
|
record.apply_report = [{"status": "skipped_not_found"}]
|
|
|
|
summary = _format_applied_edits(record)
|
|
assert summary is not None
|
|
assert "0 applied" in summary
|
|
|
|
def test_no_edits_returns_none(self) -> None:
|
|
"""无 edit 时返回 None。"""
|
|
record = MagicMock()
|
|
record.edits = []
|
|
|
|
assert _format_applied_edits(record) is None
|
|
|
|
def test_no_report_includes_all_edits(self) -> None:
|
|
"""无 apply_report 时包含所有 edit。"""
|
|
record = MagicMock()
|
|
record.edits = [{"op": "replace", "target": "foo"}]
|
|
record.apply_report = []
|
|
|
|
summary = _format_applied_edits(record)
|
|
assert summary is not None
|
|
assert "foo" in summary
|
|
|
|
|
|
class TestFallbackSummary:
|
|
"""_fallback_summary 兜底黑名单摘要。"""
|
|
|
|
def test_from_suggestions(self) -> None:
|
|
"""有 suggestions 时拼接 change 字段。"""
|
|
record = MagicMock()
|
|
record.suggestions = [{"change": "改 A"}, {"change": "改 B"}]
|
|
outcome = MagicMock()
|
|
outcome.delta_hat = -0.1
|
|
|
|
summary = _fallback_summary(record, outcome)
|
|
assert "改 A" in summary
|
|
assert "改 B" in summary
|
|
|
|
def test_no_suggestions_uses_delta(self) -> None:
|
|
"""无 suggestions 时使用 delta 信息。"""
|
|
record = MagicMock()
|
|
record.suggestions = []
|
|
outcome = MagicMock()
|
|
outcome.delta_hat = -0.15
|
|
|
|
summary = _fallback_summary(record, outcome)
|
|
assert "delta" in summary
|
|
assert "-0.15" in summary
|
|
|
|
|
|
class TestRejectedSummaryIntegration:
|
|
"""_rejected_summary_static 集成：两个子函数组合。"""
|
|
|
|
def test_static_delegates_to_format_applied(self) -> None:
|
|
"""有 applied edit 时 static 方法返回 _format_applied_edits 结果。"""
|
|
from app.harness.runner import Runner
|
|
|
|
record = MagicMock()
|
|
record.edits = [{"op": "replace", "target": "section1"}]
|
|
record.apply_report = [{"status": "applied_exact"}]
|
|
record.suggestions = []
|
|
outcome = MagicMock()
|
|
outcome.delta_hat = 0.1
|
|
|
|
summary = Runner._rejected_summary_static(record, outcome)
|
|
assert "section1" in summary
|
|
|
|
def test_static_falls_back_to_suggestions(self) -> None:
|
|
"""无 edit 时 static 方法使用 _fallback_summary。"""
|
|
from app.harness.runner import Runner
|
|
|
|
record = MagicMock()
|
|
record.edits = []
|
|
record.suggestions = [{"change": "尝试 X"}]
|
|
outcome = MagicMock()
|
|
outcome.delta_hat = -0.2
|
|
|
|
summary = Runner._rejected_summary_static(record, outcome)
|
|
assert "尝试 X" in summary
|
|
|
|
|
|
class TestWriteSkipReport:
|
|
"""_write_skip_report 辅助函数。"""
|
|
|
|
def test_writes_cooldown_report(self, tmp_path: Path) -> None:
|
|
"""cooldown 路径写 step_report JSON 文件。"""
|
|
(tmp_path / "analyses").mkdir()
|
|
_write_skip_report(
|
|
tmp_path,
|
|
epoch=1,
|
|
step=0,
|
|
global_step=5,
|
|
task_type="Action Reasoning",
|
|
action="cooldown",
|
|
baseline_acc=0.75,
|
|
budget=3,
|
|
)
|
|
report_path = tmp_path / "analyses" / "step_report_e1_s0_action-reasoning.json"
|
|
assert report_path.exists()
|
|
data = json.loads(report_path.read_text())
|
|
assert data["gate_action"] == "cooldown"
|
|
assert data["candidate_acc"] == 0.75
|
|
assert data["gate_w"] is None
|
|
|
|
def test_writes_skipped_report(self, tmp_path: Path) -> None:
|
|
"""skipped 路径写 step_report 并传递 rank_clip_triggered。"""
|
|
(tmp_path / "analyses").mkdir()
|
|
_write_skip_report(
|
|
tmp_path,
|
|
epoch=2,
|
|
step=1,
|
|
global_step=10,
|
|
task_type="Temporal Reasoning",
|
|
action="skipped",
|
|
baseline_acc=0.6,
|
|
budget=2,
|
|
rank_clip_triggered=True,
|
|
)
|
|
report_path = tmp_path / "analyses" / "step_report_e2_s1_temporal-reasoning.json"
|
|
assert report_path.exists()
|
|
data = json.loads(report_path.read_text())
|
|
assert data["gate_action"] == "skipped"
|
|
assert data["rank_clip_triggered"] is True
|
|
|
|
|
|
# =========================================================================
|
|
# 13a: _TrainState 基本构造
|
|
# =========================================================================
|
|
|
|
|
|
class TestTrainState:
|
|
"""_TrainState dataclass 基本构造与字段默认值。"""
|
|
|
|
def test_default_fields(self) -> None:
|
|
"""默认字段值正确。"""
|
|
state = _TrainState(
|
|
correctness={"q1": True},
|
|
gate_pools=MagicMock(),
|
|
baseline_cache=MagicMock(),
|
|
eval_prev_acc=0.5,
|
|
eval_prev_run_id="run1",
|
|
best_val_acc=0.5,
|
|
best_skills_version="v1",
|
|
best_prompts_version="v1",
|
|
)
|
|
assert state.global_step == 0
|
|
assert state.gate_epoch_observed is False
|
|
assert state.probations == {}
|
|
assert state.gate_cooldown == {}
|
|
assert state.rejected_buffer == {}
|
|
assert state.system_packs == []
|
|
assert state.tool_packs == []
|
|
assert state.changed_task_types_this_epoch == set()
|
|
assert state.steps_since_best_improved == 0
|
|
|
|
|
|
# =========================================================================
|
|
# 13c: cooldown 递减测试
|
|
# =========================================================================
|
|
|
|
|
|
class TestCooldownDecrement:
|
|
"""gate_cooldown 每 step 递减、归零剔除。"""
|
|
|
|
def test_decrement_and_remove(self) -> None:
|
|
"""冷却值递减，归零剔除。"""
|
|
cooldown = {"type_a": 3, "type_b": 1, "type_c": 2}
|
|
# 模拟 _run_step 末尾的冷却递减
|
|
cooldown = {t: n - 1 for t, n in cooldown.items() if n - 1 > 0}
|
|
assert cooldown == {"type_a": 2, "type_c": 1}
|
|
|
|
cooldown = {t: n - 1 for t, n in cooldown.items() if n - 1 > 0}
|
|
assert cooldown == {"type_a": 1}
|
|
|
|
cooldown = {t: n - 1 for t, n in cooldown.items() if n - 1 > 0}
|
|
assert cooldown == {}
|