"""app/harness/config.py 单元测试。 覆盖 RunConfig 构造、四层校验(_validate → 三个子校验)、 YAML + CLI + .env 三层加载优先级。 """ from __future__ import annotations from pathlib import Path import pytest import yaml from app.harness.config import RunConfig, _validate, load_config # ────────────────────────────── 测试数据工厂 ────────────────────────────── def _valid_kwargs() -> dict: """构造一组完整合法的 RunConfig 字段值(使用真实 default.yaml 数据)。""" return { "workspace_dir": Path("workspaces/default"), "store_dir": Path("store"), "mode": "infer", "run_id": "", "concurrency": 12, "max_steps": 15, "skill_mode": "auto", "n_samples": 0, "questions": "benchmarks/Video-MME", "skills_version": "v1", "prompts_version": "v1", "epochs": 1, "diag_size": 200, "diag_correct_ratio": 0.5, "val_size": 30, "val_correct_ratio": 0.5, "edit_budget_start": 5, "edit_budget_end": 2, "batch_size": 15, "min_class_per_batch": 2, "eval_min_per_class": 2, "early_stop_patience": 8, "test_size": 60, "use_slow_momentum": True, "gate_e_confirm": 20.0, "gate_e_provisional": 3.0, "gate_w_net_min": 2, "gate_delta_min": 0.02, "gate_lambda_dir": -0.642, "gate_e_rollback": 10.0, "gate_block": 8, "gate_n_max": 40, "gate_p_low": 0.05, "gate_p_high": 0.95, "gate_probe_quota": 0.2, "gate_gamma_decay": 0.9, "gate_cooldown_steps": 2, "gate_guard_err": 0.10, "skill_update_mode": "patch", "appendix_consolidate_threshold": 6, "batch_correct_ratio": 0.5, "momentum_samples": 20, } def _make_config(**overrides: object) -> RunConfig: """用 _valid_kwargs 构造 RunConfig,支持字段覆盖。""" kwargs = _valid_kwargs() kwargs.update(overrides) return RunConfig(**kwargs) def _yaml_harness_dict() -> dict: """构造可序列化为 YAML 的合法 harness 配置字典(路径用字符串)。""" d = _valid_kwargs() d["workspace_dir"] = str(d["workspace_dir"]) d["store_dir"] = str(d["store_dir"]) return d def _write_yaml(tmp_path: Path, harness_data: dict) -> Path: """将 harness 配置写入临时 YAML 文件,返回文件路径。""" yaml_path = tmp_path / "experiment.yaml" with open(yaml_path, "w", encoding="utf-8") as f: yaml.dump({"harness": harness_data}, f) return yaml_path # ──────────────────────────── test_valid_config ──────────────────────────── class TestValidConfig: """合法参数应能正常构造并通过校验。""" def test_default_yaml_values_pass_validation(self) -> None: """使用 default.yaml 真实默认值构造的 RunConfig 应通过全部校验。""" cfg = _make_config() _validate(cfg) assert cfg.mode == "infer" assert cfg.workspace_dir == Path("workspaces/default") assert cfg.store_dir == Path("store") def test_frozen_immutability(self) -> None: """frozen=True 应禁止字段赋值。""" cfg = _make_config() with pytest.raises(AttributeError): cfg.mode = "train" # type: ignore[misc] def test_default_field_values(self) -> None: """有默认值的字段不传时应使用默认值。""" kwargs = _valid_kwargs() # 不传 run_id / seed / version / resume / fresh,使用默认值 kwargs.pop("run_id", None) cfg = RunConfig(**kwargs) assert cfg.run_id == "" assert cfg.seed == "initial" assert cfg.version == "" assert cfg.resume is False assert cfg.fresh is False # ──────────────────────────── test_mode_validation ──────────────────────── class TestModeValidation: """mode 字段校验。""" @pytest.mark.parametrize("bad_mode", ["unknown", "test", "", "INFER", "Train"]) def test_invalid_mode_rejected(self, bad_mode: str) -> None: """非法 mode 应抛出 ValueError。""" cfg = _make_config(mode=bad_mode) with pytest.raises(ValueError, match="mode"): _validate(cfg) @pytest.mark.parametrize( "valid_mode", ["infer", "train", "diagnose", "evolve", "eval", "promote"] ) def test_all_valid_modes_accepted(self, valid_mode: str) -> None: """全部合法 mode 应通过校验(diagnose/evolve 需 run_id)。""" overrides: dict = {"mode": valid_mode} if valid_mode in ("diagnose", "evolve", "train"): overrides["run_id"] = "run-001" if valid_mode in ("eval", "promote"): overrides["version"] = "v1" if valid_mode == "promote": overrides["run_id"] = "run-001" cfg = _make_config(**overrides) _validate(cfg) def test_diagnose_requires_run_id(self) -> None: """diagnose 模式缺少 run_id 应报错。""" cfg = _make_config(mode="diagnose", run_id="") with pytest.raises(ValueError, match="run_id"): _validate(cfg) def test_evolve_requires_run_id(self) -> None: """evolve 模式缺少 run_id 应报错。""" cfg = _make_config(mode="evolve", run_id="") with pytest.raises(ValueError, match="run_id"): _validate(cfg) def test_eval_requires_version(self) -> None: """eval 模式缺少 version 应报错。""" cfg = _make_config(mode="eval", version="") with pytest.raises(ValueError, match="version"): _validate(cfg) def test_promote_requires_run_id_and_version(self) -> None: """promote 模式需同时提供 run_id 和 version。""" cfg = _make_config(mode="promote", run_id="", version="v1") with pytest.raises(ValueError, match="promote.*run-id"): _validate(cfg) def test_train_mode_requires_run_id_without_resume_fresh(self) -> None: """train 模式非 resume/fresh 时必须提供 run_id。""" cfg = _make_config(mode="train", run_id="", resume=False, fresh=False) with pytest.raises(ValueError, match="run_id"): _validate(cfg) def test_train_mode_resume_without_run_id_accepted(self) -> None: """train 模式 resume=True 时不需要 run_id。""" cfg = _make_config(mode="train", run_id="", resume=True) _validate(cfg) def test_train_mode_fresh_without_run_id_accepted(self) -> None: """train 模式 fresh=True 时不需要 run_id。""" cfg = _make_config(mode="train", run_id="", fresh=True) _validate(cfg) def test_train_mode_with_run_id_accepted(self) -> None: """train 模式提供 run_id 时应通过(旧式基线 run)。""" cfg = _make_config(mode="train", run_id="baseline-001") _validate(cfg) def test_concurrency_positive(self) -> None: """concurrency <= 0 应报错。""" cfg = _make_config(concurrency=0) with pytest.raises(ValueError, match="concurrency"): _validate(cfg) def test_max_steps_positive(self) -> None: """max_steps <= 0 应报错。""" cfg = _make_config(max_steps=-1) with pytest.raises(ValueError, match="max_steps"): _validate(cfg) def test_n_samples_non_negative(self) -> None: """n_samples < 0 应报错。""" cfg = _make_config(n_samples=-1) with pytest.raises(ValueError, match="n_samples"): _validate(cfg) def test_epochs_positive(self) -> None: """epochs <= 0 应报错。""" cfg = _make_config(epochs=0) with pytest.raises(ValueError, match="epochs"): _validate(cfg) @pytest.mark.parametrize("bad_skill_mode", ["Auto", "disabled", ""]) def test_invalid_skill_mode(self, bad_skill_mode: str) -> None: """非法 skill_mode 应报错。""" cfg = _make_config(skill_mode=bad_skill_mode) with pytest.raises(ValueError, match="skill_mode"): _validate(cfg) @pytest.mark.parametrize("bad_update_mode", ["append", "delete", ""]) def test_invalid_skill_update_mode(self, bad_update_mode: str) -> None: """非法 skill_update_mode 应报错。""" cfg = _make_config(skill_update_mode=bad_update_mode) with pytest.raises(ValueError, match="skill_update_mode"): _validate(cfg) def test_appendix_consolidate_threshold_positive(self) -> None: """appendix_consolidate_threshold < 1 应报错。""" cfg = _make_config(appendix_consolidate_threshold=0) with pytest.raises(ValueError, match="appendix_consolidate_threshold"): _validate(cfg) # ────────────────────── test_edit_budget_validation ─────────────────────── class TestEditBudgetValidation: """编辑预算退火校验(_validate_edit_budget)。""" def test_start_less_than_end_rejected(self) -> None: """edit_budget_start < edit_budget_end 应抛出 ValueError。""" cfg = _make_config(edit_budget_start=1, edit_budget_end=5) with pytest.raises(ValueError, match="edit_budget_start"): _validate(cfg) def test_end_zero_rejected(self) -> None: """edit_budget_end <= 0 应抛出 ValueError。""" cfg = _make_config(edit_budget_start=1, edit_budget_end=0) with pytest.raises(ValueError, match="edit_budget_end"): _validate(cfg) def test_equal_values_accepted(self) -> None: """edit_budget_start == edit_budget_end 应通过。""" cfg = _make_config(edit_budget_start=3, edit_budget_end=3) _validate(cfg) # ─────────────────────── test_minibatch_validation ─────────────────────── class TestMinibatchValidation: """mini-batch 自进化闭环参数校验(_validate_minibatch)。""" def test_batch_size_zero_rejected(self) -> None: """batch_size <= 0 应抛出 ValueError。""" cfg = _make_config(batch_size=0) with pytest.raises(ValueError, match="batch_size"): _validate(cfg) def test_min_class_per_batch_equals_batch_size_rejected(self) -> None: """min_class_per_batch >= batch_size 应抛出 ValueError。""" cfg = _make_config(batch_size=5, min_class_per_batch=5) with pytest.raises(ValueError, match="min_class_per_batch"): _validate(cfg) def test_min_class_per_batch_zero_rejected(self) -> None: """min_class_per_batch < 1 应抛出 ValueError。""" cfg = _make_config(min_class_per_batch=0) with pytest.raises(ValueError, match="min_class_per_batch"): _validate(cfg) def test_eval_min_per_class_zero_rejected(self) -> None: """eval_min_per_class < 1 应抛出 ValueError。""" cfg = _make_config(eval_min_per_class=0) with pytest.raises(ValueError, match="eval_min_per_class"): _validate(cfg) def test_early_stop_patience_zero_rejected(self) -> None: """early_stop_patience <= 0 应抛出 ValueError。""" cfg = _make_config(early_stop_patience=0) with pytest.raises(ValueError, match="early_stop_patience"): _validate(cfg) def test_test_size_zero_rejected(self) -> None: """test_size <= 0 应抛出 ValueError。""" cfg = _make_config(test_size=0) with pytest.raises(ValueError, match="test_size"): _validate(cfg) def test_batch_correct_ratio_one_rejected(self) -> None: """batch_correct_ratio >= 1 应抛出 ValueError。""" cfg = _make_config(batch_correct_ratio=1.0) with pytest.raises(ValueError, match="batch_correct_ratio"): _validate(cfg) def test_batch_correct_ratio_negative_rejected(self) -> None: """batch_correct_ratio < 0 应抛出 ValueError。""" cfg = _make_config(batch_correct_ratio=-0.1) with pytest.raises(ValueError, match="batch_correct_ratio"): _validate(cfg) def test_momentum_samples_zero_rejected(self) -> None: """momentum_samples < 1 应抛出 ValueError。""" cfg = _make_config(momentum_samples=0) with pytest.raises(ValueError, match="momentum_samples"): _validate(cfg) # ─────────────────────────── test_gate_validation ──────────────────────── class TestGateValidation: """CE-Gate 参数校验(_validate_gate)。""" def test_e_confirm_at_one_rejected(self) -> None: """gate_e_confirm <= 1 应抛出 ValueError。""" cfg = _make_config(gate_e_confirm=1.0, gate_e_provisional=1.0) with pytest.raises(ValueError, match="gate_e_confirm"): _validate(cfg) def test_e_provisional_exceeds_confirm_rejected(self) -> None: """gate_e_provisional > gate_e_confirm 应抛出 ValueError。""" cfg = _make_config(gate_e_confirm=10.0, gate_e_provisional=15.0) with pytest.raises(ValueError, match="gate_e_provisional"): _validate(cfg) def test_e_provisional_at_one_rejected(self) -> None: """gate_e_provisional <= 1 应抛出 ValueError。""" cfg = _make_config(gate_e_provisional=0.5) with pytest.raises(ValueError, match="gate_e_provisional"): _validate(cfg) def test_e_rollback_at_one_rejected(self) -> None: """gate_e_rollback <= 1 应抛出 ValueError。""" cfg = _make_config(gate_e_rollback=1.0) with pytest.raises(ValueError, match="gate_e_rollback"): _validate(cfg) def test_w_net_min_zero_rejected(self) -> None: """gate_w_net_min < 1 应抛出 ValueError。""" cfg = _make_config(gate_w_net_min=0) with pytest.raises(ValueError, match="gate_w_net_min"): _validate(cfg) def test_delta_min_negative_rejected(self) -> None: """gate_delta_min < 0 应抛出 ValueError。""" cfg = _make_config(gate_delta_min=-0.1) with pytest.raises(ValueError, match="gate_delta_min"): _validate(cfg) def test_lambda_dir_positive_rejected(self) -> None: """gate_lambda_dir >= 0 应抛出 ValueError。""" cfg = _make_config(gate_lambda_dir=0.5) with pytest.raises(ValueError, match="gate_lambda_dir"): _validate(cfg) def test_lambda_dir_zero_rejected(self) -> None: """gate_lambda_dir == 0 也应报错。""" cfg = _make_config(gate_lambda_dir=0.0) with pytest.raises(ValueError, match="gate_lambda_dir"): _validate(cfg) def test_block_exceeds_n_max_rejected(self) -> None: """gate_block > gate_n_max 应抛出 ValueError。""" cfg = _make_config(gate_block=50, gate_n_max=40) with pytest.raises(ValueError, match="gate_block"): _validate(cfg) def test_block_zero_rejected(self) -> None: """gate_block <= 0 应抛出 ValueError。""" cfg = _make_config(gate_block=0) with pytest.raises(ValueError, match="gate_block"): _validate(cfg) def test_p_low_exceeds_p_high_rejected(self) -> None: """gate_p_low >= gate_p_high 应抛出 ValueError。""" cfg = _make_config(gate_p_low=0.9, gate_p_high=0.1) with pytest.raises(ValueError, match="gate_p_low"): _validate(cfg) def test_probe_quota_negative_rejected(self) -> None: """gate_probe_quota < 0 应抛出 ValueError。""" cfg = _make_config(gate_probe_quota=-0.1) with pytest.raises(ValueError, match="gate_probe_quota"): _validate(cfg) def test_gamma_decay_zero_rejected(self) -> None: """gate_gamma_decay <= 0 应抛出 ValueError。""" cfg = _make_config(gate_gamma_decay=0.0) with pytest.raises(ValueError, match="gate_gamma_decay"): _validate(cfg) def test_gamma_decay_one_rejected(self) -> None: """gate_gamma_decay >= 1 应抛出 ValueError。""" cfg = _make_config(gate_gamma_decay=1.0) with pytest.raises(ValueError, match="gate_gamma_decay"): _validate(cfg) def test_cooldown_steps_zero_rejected(self) -> None: """gate_cooldown_steps < 1 应抛出 ValueError。""" cfg = _make_config(gate_cooldown_steps=0) with pytest.raises(ValueError, match="gate_cooldown_steps"): _validate(cfg) def test_guard_err_zero_rejected(self) -> None: """gate_guard_err <= 0 应抛出 ValueError。""" cfg = _make_config(gate_guard_err=0.0) with pytest.raises(ValueError, match="gate_guard_err"): _validate(cfg) def test_guard_err_one_rejected(self) -> None: """gate_guard_err >= 1 应抛出 ValueError。""" cfg = _make_config(gate_guard_err=1.0) with pytest.raises(ValueError, match="gate_guard_err"): _validate(cfg) # ─────────────────────── test_load_config_cli_overrides ────────────────── class TestLoadConfigCliOverrides: """load_config 的 CLI 覆盖层测试。""" def test_cli_overrides_yaml_values(self, tmp_path: Path) -> None: """CLI 参数应覆盖 YAML 中的同名字段。""" harness_data = _yaml_harness_dict() yaml_path = _write_yaml(tmp_path, harness_data) cfg = load_config(yaml_path, cli_overrides={"concurrency": 4, "max_steps": 30}) assert cfg.concurrency == 4 assert cfg.max_steps == 30 def test_cli_none_values_ignored(self, tmp_path: Path) -> None: """CLI 中值为 None 的字段不应覆盖 YAML。""" harness_data = _yaml_harness_dict() yaml_path = _write_yaml(tmp_path, harness_data) cfg = load_config(yaml_path, cli_overrides={"concurrency": None, "max_steps": 20}) assert cfg.concurrency == 12 # YAML 默认值 assert cfg.max_steps == 20 def test_cli_run_id_override(self, tmp_path: Path) -> None: """CLI 可通过 run_id 覆盖默认空字符串。""" harness_data = _yaml_harness_dict() yaml_path = _write_yaml(tmp_path, harness_data) cfg = load_config( yaml_path, cli_overrides={"mode": "diagnose", "run_id": "run-abc-123"}, ) assert cfg.run_id == "run-abc-123" assert cfg.mode == "diagnose" def test_path_fields_converted_to_path(self, tmp_path: Path) -> None: """workspace_dir 和 store_dir 应被转换为 Path 对象。""" harness_data = _yaml_harness_dict() yaml_path = _write_yaml(tmp_path, harness_data) cfg = load_config(yaml_path) assert isinstance(cfg.workspace_dir, Path) assert isinstance(cfg.store_dir, Path) def test_unknown_cli_keys_ignored(self, tmp_path: Path) -> None: """YAML 和 RunConfig 中不存在的 CLI key 应被忽略。""" harness_data = _yaml_harness_dict() yaml_path = _write_yaml(tmp_path, harness_data) cfg = load_config(yaml_path, cli_overrides={"nonexistent_field": 42}) assert cfg.concurrency == 12 # 正常字段不受影响 def test_validation_runs_after_loading(self, tmp_path: Path) -> None: """load_config 加载后应运行校验,非法值应报错。""" harness_data = _yaml_harness_dict() harness_data["mode"] = "invalid_mode" yaml_path = _write_yaml(tmp_path, harness_data) with pytest.raises(ValueError, match="mode"): load_config(yaml_path) # ──────────────────── test_load_config_env_overrides ───────────────────── class TestLoadConfigEnvOverrides: """load_config 的 .env 环境变量覆盖层测试。""" def test_env_overrides_yaml_workspace_dir( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """环境变量 HARNESS_WORKSPACE_DIR 应覆盖 YAML 中的 workspace_dir。""" harness_data = _yaml_harness_dict() yaml_path = _write_yaml(tmp_path, harness_data) monkeypatch.setenv("HARNESS_WORKSPACE_DIR", "/custom/workspace") cfg = load_config(yaml_path) assert cfg.workspace_dir == Path("/custom/workspace") def test_env_overrides_yaml_store_dir( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """环境变量 HARNESS_STORE_DIR 应覆盖 YAML 中的 store_dir。""" harness_data = _yaml_harness_dict() yaml_path = _write_yaml(tmp_path, harness_data) monkeypatch.setenv("HARNESS_STORE_DIR", "/custom/store") cfg = load_config(yaml_path) assert cfg.store_dir == Path("/custom/store") def test_cli_overrides_env(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """CLI 优先级高于 .env:CLI > .env > YAML。""" harness_data = _yaml_harness_dict() yaml_path = _write_yaml(tmp_path, harness_data) monkeypatch.setenv("HARNESS_WORKSPACE_DIR", "/env/workspace") cfg = load_config(yaml_path, cli_overrides={"workspace_dir": "/cli/workspace"}) assert cfg.workspace_dir == Path("/cli/workspace") # ──────────────────── test_load_config_real_yaml ───────────────────────── class TestLoadConfigRealYaml: """用真实 config/default.yaml 验证 load_config 嵌套解析。""" def test_load_config_real_default_yaml(self) -> None: """真实 config/default.yaml 的 harness 段应正确解析并通过校验。""" cfg = load_config(Path("config/default.yaml"), {}) assert cfg.mode == "infer" assert cfg.gate_e_confirm == 20.0 assert cfg.batch_size == 15 assert cfg.concurrency == 12 assert cfg.workspace_dir == Path("workspaces/default") assert cfg.store_dir == Path("store") assert cfg.skill_mode == "auto" def test_real_yaml_cli_override(self) -> None: """真实 YAML + CLI 覆盖应正确合并。""" cfg = load_config( Path("config/default.yaml"), {"concurrency": 4, "max_steps": 30}, ) assert cfg.concurrency == 4 assert cfg.max_steps == 30 assert cfg.mode == "infer" # 未覆盖字段保持 YAML 值 # ─────────────────────────── test_val_size_floor ───────────────────────── class TestValSizeFloor: """val_size >= eval_min_per_class * 11 的下限校验。""" def test_val_size_below_floor_rejected(self) -> None: """val_size < eval_min_per_class * 11 应抛出 ValueError。 eval_min_per_class=3 → 下限 = 3 * 11 = 33,val_size=30 不足。 """ cfg = _make_config(eval_min_per_class=3, val_size=30) with pytest.raises(ValueError, match="val_size"): _validate(cfg) def test_val_size_at_floor_accepted(self) -> None: """val_size == eval_min_per_class * 11 应通过。""" cfg = _make_config(eval_min_per_class=3, val_size=33) _validate(cfg) def test_val_size_above_floor_accepted(self) -> None: """val_size > eval_min_per_class * 11 应通过。""" cfg = _make_config(eval_min_per_class=2, val_size=100) _validate(cfg) def test_default_yaml_values_satisfy_floor(self) -> None: """default.yaml 的默认值(val_size=30, eval_min_per_class=2)应满足下限。 下限 = 2 * 11 = 22,val_size=30 >= 22,通过。 """ cfg = _make_config() _validate(cfg) # 不应抛出异常