"""五张观测表 + step/epoch 报告的单元测试。""" from __future__ import annotations import json import sqlite3 from typing import TYPE_CHECKING import pytest if TYPE_CHECKING: from pathlib import Path from app.harness.observation import ( read_dual_metric, read_gate_evidence, read_holdout_eval, read_quadrant_pairs, read_shadow_gate, write_dual_metric, write_epoch_report, write_gate_evidence, write_holdout_eval, write_quadrant_pairs, write_shadow_gate, write_step_report, ) @pytest.fixture() def db_path(tmp_path: Path) -> str: """返回临时 SQLite 路径。""" return str(tmp_path / "test_obs.db") @pytest.fixture() def run_id() -> str: return "run-obs-001" # --------------------------------------------------------------------------- # dual_metric_eval # --------------------------------------------------------------------------- def test_write_read_dual_metric(db_path: str, run_id: str) -> None: """写入 dual_metric_eval 后回读应与输入一致。""" write_dual_metric( db_path, run_id=run_id, epoch=1, version_kind="baseline", skills_version="v0", prompts_version="v0", pool="val", hard_acc=0.75, soft_score=0.80, mixed_score=0.775, ) rows = read_dual_metric(db_path, run_id=run_id) assert len(rows) == 1 row = rows[0] assert row["epoch"] == 1 assert row["version_kind"] == "baseline" assert row["skills_version"] == "v0" assert row["prompts_version"] == "v0" assert row["pool"] == "val" assert row["hard_acc"] == pytest.approx(0.75) assert row["soft_score"] == pytest.approx(0.80) assert row["mixed_score"] == pytest.approx(0.775) assert row["run_id"] == run_id # --------------------------------------------------------------------------- # shadow_gate # --------------------------------------------------------------------------- def test_write_read_shadow_gate(db_path: str, run_id: str) -> None: """写入 shadow_gate 后回读应与输入一致,is_mixed_best 布尔转 int。""" write_shadow_gate( db_path, run_id=run_id, epoch=2, candidate_version="skills/v1+prompts/v1", hard_acc=0.82, soft_score=0.78, mixed_score=0.80, is_mixed_best=True, ) rows = read_shadow_gate(db_path, run_id=run_id) assert len(rows) == 1 row = rows[0] assert row["epoch"] == 2 assert row["candidate_version"] == "skills/v1+prompts/v1" assert row["hard_acc"] == pytest.approx(0.82) assert row["is_mixed_best"] == 1 # --------------------------------------------------------------------------- # holdout_eval # --------------------------------------------------------------------------- def test_write_read_holdout_eval(db_path: str, run_id: str) -> None: """写入 holdout_eval 后回读应与输入一致。""" per_task = json.dumps({"temporal": {"accuracy": 0.9, "total": 10, "correct": 9}}) write_holdout_eval( db_path, run_id=run_id, epoch=1, version_kind="best_hard", hard_acc=0.85, soft_score=0.70, mixed_score=0.775, per_task_type_json=per_task, ) rows = read_holdout_eval(db_path, run_id=run_id) assert len(rows) == 1 row = rows[0] assert row["version_kind"] == "best_hard" assert row["hard_acc"] == pytest.approx(0.85) parsed = json.loads(row["per_task_type_json"]) assert parsed["temporal"]["correct"] == 9 # --------------------------------------------------------------------------- # gate_evidence # --------------------------------------------------------------------------- def test_write_read_gate_evidence(db_path: str, run_id: str) -> None: """写入 gate_evidence 后回读应与输入一致,逐行插入。""" evidence_rows = [ { "task_type": "temporal", "question_id": "q1", "block_idx": 0, "baseline_correct": 1, "candidate_correct": 1, "e_value": 1.0, "stop_reason": "", }, { "task_type": "temporal", "question_id": "q2", "block_idx": 0, "baseline_correct": 0, "candidate_correct": 1, "e_value": 2.0, "stop_reason": "confirmed", }, ] write_gate_evidence(db_path, run_id=run_id, epoch=1, step=0, rows=evidence_rows) rows = read_gate_evidence(db_path, run_id=run_id) assert len(rows) == 2 assert rows[0]["question_id"] == "q1" assert rows[1]["stop_reason"] == "confirmed" assert rows[1]["e_value"] == pytest.approx(2.0) # --------------------------------------------------------------------------- # quadrant_pair # --------------------------------------------------------------------------- def test_write_read_quadrant_pairs(db_path: str, run_id: str) -> None: """写入 quadrant_pair 后回读,bool -> int 转换正确。""" pairs = [ { "question_id": "q1", "task_type": "causal", "prev_correct": True, "curr_correct": False, "category": "regression", }, { "question_id": "q2", "task_type": "causal", "prev_correct": False, "curr_correct": True, "category": "improvement", }, ] write_quadrant_pairs(db_path, run_id=run_id, epoch=1, step=0, pairs=pairs) rows = read_quadrant_pairs(db_path, run_id=run_id) assert len(rows) == 2 # bool -> int 转换 assert rows[0]["prev_correct"] == 1 assert rows[0]["curr_correct"] == 0 assert rows[1]["prev_correct"] == 0 assert rows[1]["curr_correct"] == 1 # --------------------------------------------------------------------------- # NULL vs 0 语义 # --------------------------------------------------------------------------- def test_null_not_zero_for_soft(db_path: str, run_id: str) -> None: """soft_score/mixed_score 为 None 时存 NULL(非 0),回读也是 None。""" write_dual_metric( db_path, run_id=run_id, epoch=1, version_kind="baseline", skills_version="v0", prompts_version="v0", pool="val", hard_acc=0.75, soft_score=None, mixed_score=None, ) rows = read_dual_metric(db_path, run_id=run_id) assert len(rows) == 1 row = rows[0] assert row["soft_score"] is None assert row["mixed_score"] is None # 用原生 SQL 确认存的是 NULL 而非 0 conn = sqlite3.connect(db_path) cursor = conn.execute( "SELECT soft_score, mixed_score FROM dual_metric_eval WHERE run_id=?", (run_id,), ) raw = cursor.fetchone() conn.close() assert raw[0] is None assert raw[1] is None # --------------------------------------------------------------------------- # 只读连接隔离 # --------------------------------------------------------------------------- def test_read_only_connection(db_path: str, run_id: str) -> None: """read_* 使用独立只读连接,不向 _runs 表插入新行。""" write_dual_metric( db_path, run_id=run_id, epoch=1, version_kind="baseline", skills_version="v0", prompts_version="v0", pool="val", hard_acc=0.5, soft_score=None, mixed_score=None, ) # 用另一个 run_id 回读——不应在 _runs 表中创建新行 other_run = "run-obs-ghost" rows = read_dual_metric(db_path, run_id=other_run) assert rows == [] conn = sqlite3.connect(db_path) cursor = conn.execute("SELECT run_id FROM _runs") run_ids = [r[0] for r in cursor.fetchall()] conn.close() assert other_run not in run_ids # --------------------------------------------------------------------------- # step_report # --------------------------------------------------------------------------- def test_write_step_report(tmp_path: Path) -> None: """step_report 写入 JSON 文件,内容字段完整。""" workspace = tmp_path / "ws" workspace.mkdir() path = write_step_report( workspace_dir=workspace, epoch=1, step=2, global_step=12, task_type="Temporal Order", gate_action="accept_confirmed", candidate_acc=0.85, class_baseline_acc=0.70, edit_budget=5, rank_clip_triggered=False, gate_w=3, gate_l=1, gate_e_value=4.2, gate_n_used=8, gate_stop_reason="confirmed", ) assert path.exists() assert path.name == "step_report_e1_s2_temporal-order.json" data = json.loads(path.read_text(encoding="utf-8")) assert data["epoch"] == 1 assert data["step"] == 2 assert data["global_step"] == 12 assert data["task_type"] == "Temporal Order" assert data["gate_action"] == "accept_confirmed" assert data["gate_w"] == 3 assert data["gate_stop_reason"] == "confirmed" assert data["rank_clip_triggered"] is False def test_write_step_report_skipped_null_fields(tmp_path: Path) -> None: """skipped/cooldown 路径的 gate 字段应为 null。""" workspace = tmp_path / "ws" workspace.mkdir() path = write_step_report( workspace_dir=workspace, epoch=1, step=0, global_step=0, task_type="causal", gate_action="skipped", candidate_acc=0.0, class_baseline_acc=0.0, edit_budget=10, rank_clip_triggered=False, gate_w=None, gate_l=None, gate_e_value=None, gate_n_used=None, gate_stop_reason=None, ) data = json.loads(path.read_text(encoding="utf-8")) assert data["gate_w"] is None assert data["gate_l"] is None assert data["gate_e_value"] is None assert data["gate_n_used"] is None assert data["gate_stop_reason"] is None # --------------------------------------------------------------------------- # epoch_report # --------------------------------------------------------------------------- def test_write_epoch_report(tmp_path: Path) -> None: """epoch_report 写入 JSON 文件,内容字段完整。""" workspace = tmp_path / "ws" workspace.mkdir() path = write_epoch_report( workspace_dir=workspace, epoch=3, system_tool_action="updated", momentum_updated_task_types=["temporal", "causal"], best_val_acc=0.88, ) assert path.exists() assert path.name == "epoch_report_3.json" data = json.loads(path.read_text(encoding="utf-8")) assert data["epoch"] == 3 assert data["system_tool_action"] == "updated" assert data["momentum_updated_task_types"] == ["temporal", "causal"] assert data["best_val_acc"] == pytest.approx(0.88)