386 lines
12 KiB
Python
386 lines
12 KiB
Python
"""五张观测表 + step/epoch 报告的单元测试。"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import sqlite3
|
||
from typing import TYPE_CHECKING
|
||
|
||
import pytest
|
||
|
||
if TYPE_CHECKING:
|
||
from pathlib import Path
|
||
|
||
from app.harness.observation import (
|
||
read_dual_metric,
|
||
read_gate_evidence,
|
||
read_holdout_eval,
|
||
read_quadrant_pairs,
|
||
read_shadow_gate,
|
||
write_dual_metric,
|
||
write_epoch_report,
|
||
write_gate_evidence,
|
||
write_holdout_eval,
|
||
write_quadrant_pairs,
|
||
write_shadow_gate,
|
||
write_step_report,
|
||
)
|
||
|
||
|
||
@pytest.fixture()
|
||
def db_path(tmp_path: Path) -> str:
|
||
"""返回临时 SQLite 路径。"""
|
||
return str(tmp_path / "test_obs.db")
|
||
|
||
|
||
@pytest.fixture()
|
||
def run_id() -> str:
|
||
return "run-obs-001"
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# dual_metric_eval
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_write_read_dual_metric(db_path: str, run_id: str) -> None:
|
||
"""写入 dual_metric_eval 后回读应与输入一致。"""
|
||
write_dual_metric(
|
||
db_path,
|
||
run_id=run_id,
|
||
epoch=1,
|
||
version_kind="baseline",
|
||
skills_version="v0",
|
||
prompts_version="v0",
|
||
pool="val",
|
||
hard_acc=0.75,
|
||
soft_score=0.80,
|
||
mixed_score=0.775,
|
||
)
|
||
rows = read_dual_metric(db_path, run_id=run_id)
|
||
assert len(rows) == 1
|
||
row = rows[0]
|
||
assert row["epoch"] == 1
|
||
assert row["version_kind"] == "baseline"
|
||
assert row["skills_version"] == "v0"
|
||
assert row["prompts_version"] == "v0"
|
||
assert row["pool"] == "val"
|
||
assert row["hard_acc"] == pytest.approx(0.75)
|
||
assert row["soft_score"] == pytest.approx(0.80)
|
||
assert row["mixed_score"] == pytest.approx(0.775)
|
||
assert row["run_id"] == run_id
|
||
|
||
|
||
def test_dual_metric_kinds_distinguishable(db_path: str, run_id: str) -> None:
|
||
"""observation 层接受任意 version_kind：final 与 slow_candidate 可区分共存。"""
|
||
write_dual_metric(
|
||
db_path,
|
||
run_id=run_id,
|
||
epoch=1,
|
||
version_kind="final",
|
||
skills_version="v1",
|
||
prompts_version="v1",
|
||
pool="val",
|
||
hard_acc=0.7,
|
||
soft_score=None,
|
||
mixed_score=None,
|
||
)
|
||
write_dual_metric(
|
||
db_path,
|
||
run_id=run_id,
|
||
epoch=1,
|
||
version_kind="slow_candidate",
|
||
skills_version="v2",
|
||
prompts_version="v2",
|
||
pool="val",
|
||
hard_acc=0.6,
|
||
soft_score=None,
|
||
mixed_score=None,
|
||
)
|
||
rows = read_dual_metric(db_path, run_id=run_id)
|
||
kinds = {row["version_kind"] for row in rows if row["epoch"] == 1}
|
||
assert kinds == {"final", "slow_candidate"}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# shadow_gate
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_write_read_shadow_gate(db_path: str, run_id: str) -> None:
|
||
"""写入 shadow_gate 后回读应与输入一致，is_mixed_best 布尔转 int。"""
|
||
write_shadow_gate(
|
||
db_path,
|
||
run_id=run_id,
|
||
epoch=2,
|
||
candidate_version="skills/v1+prompts/v1",
|
||
hard_acc=0.82,
|
||
soft_score=0.78,
|
||
mixed_score=0.80,
|
||
is_mixed_best=True,
|
||
)
|
||
rows = read_shadow_gate(db_path, run_id=run_id)
|
||
assert len(rows) == 1
|
||
row = rows[0]
|
||
assert row["epoch"] == 2
|
||
assert row["candidate_version"] == "skills/v1+prompts/v1"
|
||
assert row["hard_acc"] == pytest.approx(0.82)
|
||
assert row["is_mixed_best"] == 1
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# holdout_eval
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_write_read_holdout_eval(db_path: str, run_id: str) -> None:
|
||
"""写入 holdout_eval 后回读应与输入一致。"""
|
||
per_task = json.dumps({"temporal": {"accuracy": 0.9, "total": 10, "correct": 9}})
|
||
write_holdout_eval(
|
||
db_path,
|
||
run_id=run_id,
|
||
epoch=1,
|
||
version_kind="best_hard",
|
||
hard_acc=0.85,
|
||
soft_score=0.70,
|
||
mixed_score=0.775,
|
||
per_task_type_json=per_task,
|
||
)
|
||
rows = read_holdout_eval(db_path, run_id=run_id)
|
||
assert len(rows) == 1
|
||
row = rows[0]
|
||
assert row["version_kind"] == "best_hard"
|
||
assert row["hard_acc"] == pytest.approx(0.85)
|
||
parsed = json.loads(row["per_task_type_json"])
|
||
assert parsed["temporal"]["correct"] == 9
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# gate_evidence
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_write_read_gate_evidence(db_path: str, run_id: str) -> None:
|
||
"""写入 gate_evidence 后回读应与输入一致，逐行插入。"""
|
||
evidence_rows = [
|
||
{
|
||
"task_type": "temporal",
|
||
"question_id": "q1",
|
||
"block_idx": 0,
|
||
"baseline_correct": 1,
|
||
"candidate_correct": 1,
|
||
"e_value": 1.0,
|
||
"stop_reason": "",
|
||
},
|
||
{
|
||
"task_type": "temporal",
|
||
"question_id": "q2",
|
||
"block_idx": 0,
|
||
"baseline_correct": 0,
|
||
"candidate_correct": 1,
|
||
"e_value": 2.0,
|
||
"stop_reason": "confirmed",
|
||
},
|
||
]
|
||
write_gate_evidence(db_path, run_id=run_id, epoch=1, step=0, rows=evidence_rows)
|
||
rows = read_gate_evidence(db_path, run_id=run_id)
|
||
assert len(rows) == 2
|
||
assert rows[0]["question_id"] == "q1"
|
||
assert rows[1]["stop_reason"] == "confirmed"
|
||
assert rows[1]["e_value"] == pytest.approx(2.0)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# quadrant_pair
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_write_read_quadrant_pairs(db_path: str, run_id: str) -> None:
|
||
"""写入 quadrant_pair 后回读，bool -> int 转换正确。"""
|
||
pairs = [
|
||
{
|
||
"question_id": "q1",
|
||
"task_type": "causal",
|
||
"prev_correct": True,
|
||
"curr_correct": False,
|
||
"category": "regression",
|
||
},
|
||
{
|
||
"question_id": "q2",
|
||
"task_type": "causal",
|
||
"prev_correct": False,
|
||
"curr_correct": True,
|
||
"category": "improvement",
|
||
},
|
||
]
|
||
write_quadrant_pairs(db_path, run_id=run_id, epoch=1, step=0, pairs=pairs)
|
||
rows = read_quadrant_pairs(db_path, run_id=run_id)
|
||
assert len(rows) == 2
|
||
# bool -> int 转换
|
||
assert rows[0]["prev_correct"] == 1
|
||
assert rows[0]["curr_correct"] == 0
|
||
assert rows[1]["prev_correct"] == 0
|
||
assert rows[1]["curr_correct"] == 1
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# NULL vs 0 语义
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_null_not_zero_for_soft(db_path: str, run_id: str) -> None:
|
||
"""soft_score/mixed_score 为 None 时存 NULL（非 0），回读也是 None。"""
|
||
write_dual_metric(
|
||
db_path,
|
||
run_id=run_id,
|
||
epoch=1,
|
||
version_kind="baseline",
|
||
skills_version="v0",
|
||
prompts_version="v0",
|
||
pool="val",
|
||
hard_acc=0.75,
|
||
soft_score=None,
|
||
mixed_score=None,
|
||
)
|
||
rows = read_dual_metric(db_path, run_id=run_id)
|
||
assert len(rows) == 1
|
||
row = rows[0]
|
||
assert row["soft_score"] is None
|
||
assert row["mixed_score"] is None
|
||
|
||
# 用原生 SQL 确认存的是 NULL 而非 0
|
||
conn = sqlite3.connect(db_path)
|
||
cursor = conn.execute(
|
||
"SELECT soft_score, mixed_score FROM dual_metric_eval WHERE run_id=?",
|
||
(run_id,),
|
||
)
|
||
raw = cursor.fetchone()
|
||
conn.close()
|
||
assert raw[0] is None
|
||
assert raw[1] is None
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 只读连接隔离
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_read_only_connection(db_path: str, run_id: str) -> None:
|
||
"""read_* 使用独立只读连接，不向 _runs 表插入新行。"""
|
||
write_dual_metric(
|
||
db_path,
|
||
run_id=run_id,
|
||
epoch=1,
|
||
version_kind="baseline",
|
||
skills_version="v0",
|
||
prompts_version="v0",
|
||
pool="val",
|
||
hard_acc=0.5,
|
||
soft_score=None,
|
||
mixed_score=None,
|
||
)
|
||
|
||
# 用另一个 run_id 回读——不应在 _runs 表中创建新行
|
||
other_run = "run-obs-ghost"
|
||
rows = read_dual_metric(db_path, run_id=other_run)
|
||
assert rows == []
|
||
|
||
conn = sqlite3.connect(db_path)
|
||
cursor = conn.execute("SELECT run_id FROM _runs")
|
||
run_ids = [r[0] for r in cursor.fetchall()]
|
||
conn.close()
|
||
assert other_run not in run_ids
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# step_report
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_write_step_report(tmp_path: Path) -> None:
|
||
"""step_report 写入 JSON 文件，内容字段完整。"""
|
||
workspace = tmp_path / "ws"
|
||
workspace.mkdir()
|
||
path = write_step_report(
|
||
workspace_dir=workspace,
|
||
epoch=1,
|
||
step=2,
|
||
global_step=12,
|
||
task_type="Temporal Order",
|
||
gate_action="accept_confirmed",
|
||
candidate_acc=0.85,
|
||
class_baseline_acc=0.70,
|
||
edit_budget=5,
|
||
rank_clip_triggered=False,
|
||
gate_w=3,
|
||
gate_l=1,
|
||
gate_e_value=4.2,
|
||
gate_n_used=8,
|
||
gate_stop_reason="confirmed",
|
||
)
|
||
assert path.exists()
|
||
assert path.name == "step_report_e1_s2_temporal-order.json"
|
||
data = json.loads(path.read_text(encoding="utf-8"))
|
||
assert data["epoch"] == 1
|
||
assert data["step"] == 2
|
||
assert data["global_step"] == 12
|
||
assert data["task_type"] == "Temporal Order"
|
||
assert data["gate_action"] == "accept_confirmed"
|
||
assert data["gate_w"] == 3
|
||
assert data["gate_stop_reason"] == "confirmed"
|
||
assert data["rank_clip_triggered"] is False
|
||
|
||
|
||
def test_write_step_report_skipped_null_fields(tmp_path: Path) -> None:
|
||
"""skipped/cooldown 路径的 gate 字段应为 null。"""
|
||
workspace = tmp_path / "ws"
|
||
workspace.mkdir()
|
||
path = write_step_report(
|
||
workspace_dir=workspace,
|
||
epoch=1,
|
||
step=0,
|
||
global_step=0,
|
||
task_type="causal",
|
||
gate_action="skipped",
|
||
candidate_acc=0.0,
|
||
class_baseline_acc=0.0,
|
||
edit_budget=10,
|
||
rank_clip_triggered=False,
|
||
gate_w=None,
|
||
gate_l=None,
|
||
gate_e_value=None,
|
||
gate_n_used=None,
|
||
gate_stop_reason=None,
|
||
)
|
||
data = json.loads(path.read_text(encoding="utf-8"))
|
||
assert data["gate_w"] is None
|
||
assert data["gate_l"] is None
|
||
assert data["gate_e_value"] is None
|
||
assert data["gate_n_used"] is None
|
||
assert data["gate_stop_reason"] is None
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# epoch_report
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_write_epoch_report(tmp_path: Path) -> None:
|
||
"""epoch_report 写入 JSON 文件，内容字段完整。"""
|
||
workspace = tmp_path / "ws"
|
||
workspace.mkdir()
|
||
path = write_epoch_report(
|
||
workspace_dir=workspace,
|
||
epoch=3,
|
||
system_tool_action="updated",
|
||
momentum_updated_task_types=["temporal", "causal"],
|
||
best_val_acc=0.88,
|
||
)
|
||
assert path.exists()
|
||
assert path.name == "epoch_report_3.json"
|
||
data = json.loads(path.read_text(encoding="utf-8"))
|
||
assert data["epoch"] == 3
|
||
assert data["system_tool_action"] == "updated"
|
||
assert data["momentum_updated_task_types"] == ["temporal", "causal"]
|
||
assert data["best_val_acc"] == pytest.approx(0.88)
|