feat(harness): observation.py — 五张观测表 + step/epoch 报告

2026-07-07 12:55:12 -04:00
parent bd4e438c6c
commit a550d39e1c
2 changed files with 813 additions and 0 deletions
@@ -0,0 +1,354 @@
+"""五张观测表 + step/epoch 报告的单元测试。"""
+
+from __future__ import annotations
+
+import json
+import sqlite3
+from typing import TYPE_CHECKING
+
+import pytest
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+from app.harness.observation import (
+    read_dual_metric,
+    read_gate_evidence,
+    read_holdout_eval,
+    read_quadrant_pairs,
+    read_shadow_gate,
+    write_dual_metric,
+    write_epoch_report,
+    write_gate_evidence,
+    write_holdout_eval,
+    write_quadrant_pairs,
+    write_shadow_gate,
+    write_step_report,
+)
+
+
+@pytest.fixture()
+def db_path(tmp_path: Path) -> str:
+    """返回临时 SQLite 路径。"""
+    return str(tmp_path / "test_obs.db")
+
+
+@pytest.fixture()
+def run_id() -> str:
+    return "run-obs-001"
+
+
+# ---------------------------------------------------------------------------
+# dual_metric_eval
+# ---------------------------------------------------------------------------
+
+
+def test_write_read_dual_metric(db_path: str, run_id: str) -> None:
+    """写入 dual_metric_eval 后回读应与输入一致。"""
+    write_dual_metric(
+        db_path,
+        run_id=run_id,
+        epoch=1,
+        version_kind="baseline",
+        skills_version="v0",
+        prompts_version="v0",
+        pool="val",
+        hard_acc=0.75,
+        soft_score=0.80,
+        mixed_score=0.775,
+    )
+    rows = read_dual_metric(db_path, run_id=run_id)
+    assert len(rows) == 1
+    row = rows[0]
+    assert row["epoch"] == 1
+    assert row["version_kind"] == "baseline"
+    assert row["skills_version"] == "v0"
+    assert row["prompts_version"] == "v0"
+    assert row["pool"] == "val"
+    assert row["hard_acc"] == pytest.approx(0.75)
+    assert row["soft_score"] == pytest.approx(0.80)
+    assert row["mixed_score"] == pytest.approx(0.775)
+    assert row["run_id"] == run_id
+
+
+# ---------------------------------------------------------------------------
+# shadow_gate
+# ---------------------------------------------------------------------------
+
+
+def test_write_read_shadow_gate(db_path: str, run_id: str) -> None:
+    """写入 shadow_gate 后回读应与输入一致，is_mixed_best 布尔转 int。"""
+    write_shadow_gate(
+        db_path,
+        run_id=run_id,
+        epoch=2,
+        candidate_version="skills/v1+prompts/v1",
+        hard_acc=0.82,
+        soft_score=0.78,
+        mixed_score=0.80,
+        is_mixed_best=True,
+    )
+    rows = read_shadow_gate(db_path, run_id=run_id)
+    assert len(rows) == 1
+    row = rows[0]
+    assert row["epoch"] == 2
+    assert row["candidate_version"] == "skills/v1+prompts/v1"
+    assert row["hard_acc"] == pytest.approx(0.82)
+    assert row["is_mixed_best"] == 1
+
+
+# ---------------------------------------------------------------------------
+# holdout_eval
+# ---------------------------------------------------------------------------
+
+
+def test_write_read_holdout_eval(db_path: str, run_id: str) -> None:
+    """写入 holdout_eval 后回读应与输入一致。"""
+    per_task = json.dumps({"temporal": {"accuracy": 0.9, "total": 10, "correct": 9}})
+    write_holdout_eval(
+        db_path,
+        run_id=run_id,
+        epoch=1,
+        version_kind="best_hard",
+        hard_acc=0.85,
+        soft_score=0.70,
+        mixed_score=0.775,
+        per_task_type_json=per_task,
+    )
+    rows = read_holdout_eval(db_path, run_id=run_id)
+    assert len(rows) == 1
+    row = rows[0]
+    assert row["version_kind"] == "best_hard"
+    assert row["hard_acc"] == pytest.approx(0.85)
+    parsed = json.loads(row["per_task_type_json"])
+    assert parsed["temporal"]["correct"] == 9
+
+
+# ---------------------------------------------------------------------------
+# gate_evidence
+# ---------------------------------------------------------------------------
+
+
+def test_write_read_gate_evidence(db_path: str, run_id: str) -> None:
+    """写入 gate_evidence 后回读应与输入一致，逐行插入。"""
+    evidence_rows = [
+        {
+            "task_type": "temporal",
+            "question_id": "q1",
+            "block_idx": 0,
+            "baseline_correct": 1,
+            "candidate_correct": 1,
+            "e_value": 1.0,
+            "stop_reason": "",
+        },
+        {
+            "task_type": "temporal",
+            "question_id": "q2",
+            "block_idx": 0,
+            "baseline_correct": 0,
+            "candidate_correct": 1,
+            "e_value": 2.0,
+            "stop_reason": "confirmed",
+        },
+    ]
+    write_gate_evidence(db_path, run_id=run_id, epoch=1, step=0, rows=evidence_rows)
+    rows = read_gate_evidence(db_path, run_id=run_id)
+    assert len(rows) == 2
+    assert rows[0]["question_id"] == "q1"
+    assert rows[1]["stop_reason"] == "confirmed"
+    assert rows[1]["e_value"] == pytest.approx(2.0)
+
+
+# ---------------------------------------------------------------------------
+# quadrant_pair
+# ---------------------------------------------------------------------------
+
+
+def test_write_read_quadrant_pairs(db_path: str, run_id: str) -> None:
+    """写入 quadrant_pair 后回读，bool -> int 转换正确。"""
+    pairs = [
+        {
+            "question_id": "q1",
+            "task_type": "causal",
+            "prev_correct": True,
+            "curr_correct": False,
+            "category": "regression",
+        },
+        {
+            "question_id": "q2",
+            "task_type": "causal",
+            "prev_correct": False,
+            "curr_correct": True,
+            "category": "improvement",
+        },
+    ]
+    write_quadrant_pairs(db_path, run_id=run_id, epoch=1, step=0, pairs=pairs)
+    rows = read_quadrant_pairs(db_path, run_id=run_id)
+    assert len(rows) == 2
+    # bool -> int 转换
+    assert rows[0]["prev_correct"] == 1
+    assert rows[0]["curr_correct"] == 0
+    assert rows[1]["prev_correct"] == 0
+    assert rows[1]["curr_correct"] == 1
+
+
+# ---------------------------------------------------------------------------
+# NULL vs 0 语义
+# ---------------------------------------------------------------------------
+
+
+def test_null_not_zero_for_soft(db_path: str, run_id: str) -> None:
+    """soft_score/mixed_score 为 None 时存 NULL（非 0），回读也是 None。"""
+    write_dual_metric(
+        db_path,
+        run_id=run_id,
+        epoch=1,
+        version_kind="baseline",
+        skills_version="v0",
+        prompts_version="v0",
+        pool="val",
+        hard_acc=0.75,
+        soft_score=None,
+        mixed_score=None,
+    )
+    rows = read_dual_metric(db_path, run_id=run_id)
+    assert len(rows) == 1
+    row = rows[0]
+    assert row["soft_score"] is None
+    assert row["mixed_score"] is None
+
+    # 用原生 SQL 确认存的是 NULL 而非 0
+    conn = sqlite3.connect(db_path)
+    cursor = conn.execute(
+        "SELECT soft_score, mixed_score FROM dual_metric_eval WHERE run_id=?",
+        (run_id,),
+    )
+    raw = cursor.fetchone()
+    conn.close()
+    assert raw[0] is None
+    assert raw[1] is None
+
+
+# ---------------------------------------------------------------------------
+# 只读连接隔离
+# ---------------------------------------------------------------------------
+
+
+def test_read_only_connection(db_path: str, run_id: str) -> None:
+    """read_* 使用独立只读连接，不向 _runs 表插入新行。"""
+    write_dual_metric(
+        db_path,
+        run_id=run_id,
+        epoch=1,
+        version_kind="baseline",
+        skills_version="v0",
+        prompts_version="v0",
+        pool="val",
+        hard_acc=0.5,
+        soft_score=None,
+        mixed_score=None,
+    )
+
+    # 用另一个 run_id 回读——不应在 _runs 表中创建新行
+    other_run = "run-obs-ghost"
+    rows = read_dual_metric(db_path, run_id=other_run)
+    assert rows == []
+
+    conn = sqlite3.connect(db_path)
+    cursor = conn.execute("SELECT run_id FROM _runs")
+    run_ids = [r[0] for r in cursor.fetchall()]
+    conn.close()
+    assert other_run not in run_ids
+
+
+# ---------------------------------------------------------------------------
+# step_report
+# ---------------------------------------------------------------------------
+
+
+def test_write_step_report(tmp_path: Path) -> None:
+    """step_report 写入 JSON 文件，内容字段完整。"""
+    workspace = tmp_path / "ws"
+    workspace.mkdir()
+    path = write_step_report(
+        workspace_dir=workspace,
+        epoch=1,
+        step=2,
+        global_step=12,
+        task_type="Temporal Order",
+        gate_action="accept_confirmed",
+        candidate_acc=0.85,
+        class_baseline_acc=0.70,
+        edit_budget=5,
+        rank_clip_triggered=False,
+        gate_w=3,
+        gate_l=1,
+        gate_e_value=4.2,
+        gate_n_used=8,
+        gate_stop_reason="confirmed",
+    )
+    assert path.exists()
+    assert path.name == "step_report_e1_s2_temporal-order.json"
+    data = json.loads(path.read_text(encoding="utf-8"))
+    assert data["epoch"] == 1
+    assert data["step"] == 2
+    assert data["global_step"] == 12
+    assert data["task_type"] == "Temporal Order"
+    assert data["gate_action"] == "accept_confirmed"
+    assert data["gate_w"] == 3
+    assert data["gate_stop_reason"] == "confirmed"
+    assert data["rank_clip_triggered"] is False
+
+
+def test_write_step_report_skipped_null_fields(tmp_path: Path) -> None:
+    """skipped/cooldown 路径的 gate 字段应为 null。"""
+    workspace = tmp_path / "ws"
+    workspace.mkdir()
+    path = write_step_report(
+        workspace_dir=workspace,
+        epoch=1,
+        step=0,
+        global_step=0,
+        task_type="causal",
+        gate_action="skipped",
+        candidate_acc=0.0,
+        class_baseline_acc=0.0,
+        edit_budget=10,
+        rank_clip_triggered=False,
+        gate_w=None,
+        gate_l=None,
+        gate_e_value=None,
+        gate_n_used=None,
+        gate_stop_reason=None,
+    )
+    data = json.loads(path.read_text(encoding="utf-8"))
+    assert data["gate_w"] is None
+    assert data["gate_l"] is None
+    assert data["gate_e_value"] is None
+    assert data["gate_n_used"] is None
+    assert data["gate_stop_reason"] is None
+
+
+# ---------------------------------------------------------------------------
+# epoch_report
+# ---------------------------------------------------------------------------
+
+
+def test_write_epoch_report(tmp_path: Path) -> None:
+    """epoch_report 写入 JSON 文件，内容字段完整。"""
+    workspace = tmp_path / "ws"
+    workspace.mkdir()
+    path = write_epoch_report(
+        workspace_dir=workspace,
+        epoch=3,
+        system_tool_action="updated",
+        momentum_updated_task_types=["temporal", "causal"],
+        best_val_acc=0.88,
+    )
+    assert path.exists()
+    assert path.name == "epoch_report_3.json"
+    data = json.loads(path.read_text(encoding="utf-8"))
+    assert data["epoch"] == 3
+    assert data["system_tool_action"] == "updated"
+    assert data["momentum_updated_task_types"] == ["temporal", "causal"]
+    assert data["best_val_acc"] == pytest.approx(0.88)