355 lines
11 KiB
Python
355 lines
11 KiB
Python
"""五张观测表 + step/epoch 报告的单元测试。"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sqlite3
|
|
from typing import TYPE_CHECKING
|
|
|
|
import pytest
|
|
|
|
if TYPE_CHECKING:
|
|
from pathlib import Path
|
|
|
|
from app.harness.observation import (
|
|
read_dual_metric,
|
|
read_gate_evidence,
|
|
read_holdout_eval,
|
|
read_quadrant_pairs,
|
|
read_shadow_gate,
|
|
write_dual_metric,
|
|
write_epoch_report,
|
|
write_gate_evidence,
|
|
write_holdout_eval,
|
|
write_quadrant_pairs,
|
|
write_shadow_gate,
|
|
write_step_report,
|
|
)
|
|
|
|
|
|
@pytest.fixture()
|
|
def db_path(tmp_path: Path) -> str:
|
|
"""返回临时 SQLite 路径。"""
|
|
return str(tmp_path / "test_obs.db")
|
|
|
|
|
|
@pytest.fixture()
|
|
def run_id() -> str:
|
|
return "run-obs-001"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# dual_metric_eval
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_write_read_dual_metric(db_path: str, run_id: str) -> None:
|
|
"""写入 dual_metric_eval 后回读应与输入一致。"""
|
|
write_dual_metric(
|
|
db_path,
|
|
run_id=run_id,
|
|
epoch=1,
|
|
version_kind="baseline",
|
|
skills_version="v0",
|
|
prompts_version="v0",
|
|
pool="val",
|
|
hard_acc=0.75,
|
|
soft_score=0.80,
|
|
mixed_score=0.775,
|
|
)
|
|
rows = read_dual_metric(db_path, run_id=run_id)
|
|
assert len(rows) == 1
|
|
row = rows[0]
|
|
assert row["epoch"] == 1
|
|
assert row["version_kind"] == "baseline"
|
|
assert row["skills_version"] == "v0"
|
|
assert row["prompts_version"] == "v0"
|
|
assert row["pool"] == "val"
|
|
assert row["hard_acc"] == pytest.approx(0.75)
|
|
assert row["soft_score"] == pytest.approx(0.80)
|
|
assert row["mixed_score"] == pytest.approx(0.775)
|
|
assert row["run_id"] == run_id
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# shadow_gate
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_write_read_shadow_gate(db_path: str, run_id: str) -> None:
|
|
"""写入 shadow_gate 后回读应与输入一致，is_mixed_best 布尔转 int。"""
|
|
write_shadow_gate(
|
|
db_path,
|
|
run_id=run_id,
|
|
epoch=2,
|
|
candidate_version="skills/v1+prompts/v1",
|
|
hard_acc=0.82,
|
|
soft_score=0.78,
|
|
mixed_score=0.80,
|
|
is_mixed_best=True,
|
|
)
|
|
rows = read_shadow_gate(db_path, run_id=run_id)
|
|
assert len(rows) == 1
|
|
row = rows[0]
|
|
assert row["epoch"] == 2
|
|
assert row["candidate_version"] == "skills/v1+prompts/v1"
|
|
assert row["hard_acc"] == pytest.approx(0.82)
|
|
assert row["is_mixed_best"] == 1
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# holdout_eval
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_write_read_holdout_eval(db_path: str, run_id: str) -> None:
|
|
"""写入 holdout_eval 后回读应与输入一致。"""
|
|
per_task = json.dumps({"temporal": {"accuracy": 0.9, "total": 10, "correct": 9}})
|
|
write_holdout_eval(
|
|
db_path,
|
|
run_id=run_id,
|
|
epoch=1,
|
|
version_kind="best_hard",
|
|
hard_acc=0.85,
|
|
soft_score=0.70,
|
|
mixed_score=0.775,
|
|
per_task_type_json=per_task,
|
|
)
|
|
rows = read_holdout_eval(db_path, run_id=run_id)
|
|
assert len(rows) == 1
|
|
row = rows[0]
|
|
assert row["version_kind"] == "best_hard"
|
|
assert row["hard_acc"] == pytest.approx(0.85)
|
|
parsed = json.loads(row["per_task_type_json"])
|
|
assert parsed["temporal"]["correct"] == 9
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# gate_evidence
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_write_read_gate_evidence(db_path: str, run_id: str) -> None:
|
|
"""写入 gate_evidence 后回读应与输入一致，逐行插入。"""
|
|
evidence_rows = [
|
|
{
|
|
"task_type": "temporal",
|
|
"question_id": "q1",
|
|
"block_idx": 0,
|
|
"baseline_correct": 1,
|
|
"candidate_correct": 1,
|
|
"e_value": 1.0,
|
|
"stop_reason": "",
|
|
},
|
|
{
|
|
"task_type": "temporal",
|
|
"question_id": "q2",
|
|
"block_idx": 0,
|
|
"baseline_correct": 0,
|
|
"candidate_correct": 1,
|
|
"e_value": 2.0,
|
|
"stop_reason": "confirmed",
|
|
},
|
|
]
|
|
write_gate_evidence(db_path, run_id=run_id, epoch=1, step=0, rows=evidence_rows)
|
|
rows = read_gate_evidence(db_path, run_id=run_id)
|
|
assert len(rows) == 2
|
|
assert rows[0]["question_id"] == "q1"
|
|
assert rows[1]["stop_reason"] == "confirmed"
|
|
assert rows[1]["e_value"] == pytest.approx(2.0)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# quadrant_pair
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_write_read_quadrant_pairs(db_path: str, run_id: str) -> None:
|
|
"""写入 quadrant_pair 后回读，bool -> int 转换正确。"""
|
|
pairs = [
|
|
{
|
|
"question_id": "q1",
|
|
"task_type": "causal",
|
|
"prev_correct": True,
|
|
"curr_correct": False,
|
|
"category": "regression",
|
|
},
|
|
{
|
|
"question_id": "q2",
|
|
"task_type": "causal",
|
|
"prev_correct": False,
|
|
"curr_correct": True,
|
|
"category": "improvement",
|
|
},
|
|
]
|
|
write_quadrant_pairs(db_path, run_id=run_id, epoch=1, step=0, pairs=pairs)
|
|
rows = read_quadrant_pairs(db_path, run_id=run_id)
|
|
assert len(rows) == 2
|
|
# bool -> int 转换
|
|
assert rows[0]["prev_correct"] == 1
|
|
assert rows[0]["curr_correct"] == 0
|
|
assert rows[1]["prev_correct"] == 0
|
|
assert rows[1]["curr_correct"] == 1
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# NULL vs 0 语义
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_null_not_zero_for_soft(db_path: str, run_id: str) -> None:
|
|
"""soft_score/mixed_score 为 None 时存 NULL（非 0），回读也是 None。"""
|
|
write_dual_metric(
|
|
db_path,
|
|
run_id=run_id,
|
|
epoch=1,
|
|
version_kind="baseline",
|
|
skills_version="v0",
|
|
prompts_version="v0",
|
|
pool="val",
|
|
hard_acc=0.75,
|
|
soft_score=None,
|
|
mixed_score=None,
|
|
)
|
|
rows = read_dual_metric(db_path, run_id=run_id)
|
|
assert len(rows) == 1
|
|
row = rows[0]
|
|
assert row["soft_score"] is None
|
|
assert row["mixed_score"] is None
|
|
|
|
# 用原生 SQL 确认存的是 NULL 而非 0
|
|
conn = sqlite3.connect(db_path)
|
|
cursor = conn.execute(
|
|
"SELECT soft_score, mixed_score FROM dual_metric_eval WHERE run_id=?",
|
|
(run_id,),
|
|
)
|
|
raw = cursor.fetchone()
|
|
conn.close()
|
|
assert raw[0] is None
|
|
assert raw[1] is None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 只读连接隔离
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_read_only_connection(db_path: str, run_id: str) -> None:
|
|
"""read_* 使用独立只读连接，不向 _runs 表插入新行。"""
|
|
write_dual_metric(
|
|
db_path,
|
|
run_id=run_id,
|
|
epoch=1,
|
|
version_kind="baseline",
|
|
skills_version="v0",
|
|
prompts_version="v0",
|
|
pool="val",
|
|
hard_acc=0.5,
|
|
soft_score=None,
|
|
mixed_score=None,
|
|
)
|
|
|
|
# 用另一个 run_id 回读——不应在 _runs 表中创建新行
|
|
other_run = "run-obs-ghost"
|
|
rows = read_dual_metric(db_path, run_id=other_run)
|
|
assert rows == []
|
|
|
|
conn = sqlite3.connect(db_path)
|
|
cursor = conn.execute("SELECT run_id FROM _runs")
|
|
run_ids = [r[0] for r in cursor.fetchall()]
|
|
conn.close()
|
|
assert other_run not in run_ids
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# step_report
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_write_step_report(tmp_path: Path) -> None:
|
|
"""step_report 写入 JSON 文件，内容字段完整。"""
|
|
workspace = tmp_path / "ws"
|
|
workspace.mkdir()
|
|
path = write_step_report(
|
|
workspace_dir=workspace,
|
|
epoch=1,
|
|
step=2,
|
|
global_step=12,
|
|
task_type="Temporal Order",
|
|
gate_action="accept_confirmed",
|
|
candidate_acc=0.85,
|
|
class_baseline_acc=0.70,
|
|
edit_budget=5,
|
|
rank_clip_triggered=False,
|
|
gate_w=3,
|
|
gate_l=1,
|
|
gate_e_value=4.2,
|
|
gate_n_used=8,
|
|
gate_stop_reason="confirmed",
|
|
)
|
|
assert path.exists()
|
|
assert path.name == "step_report_e1_s2_temporal-order.json"
|
|
data = json.loads(path.read_text(encoding="utf-8"))
|
|
assert data["epoch"] == 1
|
|
assert data["step"] == 2
|
|
assert data["global_step"] == 12
|
|
assert data["task_type"] == "Temporal Order"
|
|
assert data["gate_action"] == "accept_confirmed"
|
|
assert data["gate_w"] == 3
|
|
assert data["gate_stop_reason"] == "confirmed"
|
|
assert data["rank_clip_triggered"] is False
|
|
|
|
|
|
def test_write_step_report_skipped_null_fields(tmp_path: Path) -> None:
|
|
"""skipped/cooldown 路径的 gate 字段应为 null。"""
|
|
workspace = tmp_path / "ws"
|
|
workspace.mkdir()
|
|
path = write_step_report(
|
|
workspace_dir=workspace,
|
|
epoch=1,
|
|
step=0,
|
|
global_step=0,
|
|
task_type="causal",
|
|
gate_action="skipped",
|
|
candidate_acc=0.0,
|
|
class_baseline_acc=0.0,
|
|
edit_budget=10,
|
|
rank_clip_triggered=False,
|
|
gate_w=None,
|
|
gate_l=None,
|
|
gate_e_value=None,
|
|
gate_n_used=None,
|
|
gate_stop_reason=None,
|
|
)
|
|
data = json.loads(path.read_text(encoding="utf-8"))
|
|
assert data["gate_w"] is None
|
|
assert data["gate_l"] is None
|
|
assert data["gate_e_value"] is None
|
|
assert data["gate_n_used"] is None
|
|
assert data["gate_stop_reason"] is None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# epoch_report
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_write_epoch_report(tmp_path: Path) -> None:
|
|
"""epoch_report 写入 JSON 文件，内容字段完整。"""
|
|
workspace = tmp_path / "ws"
|
|
workspace.mkdir()
|
|
path = write_epoch_report(
|
|
workspace_dir=workspace,
|
|
epoch=3,
|
|
system_tool_action="updated",
|
|
momentum_updated_task_types=["temporal", "causal"],
|
|
best_val_acc=0.88,
|
|
)
|
|
assert path.exists()
|
|
assert path.name == "epoch_report_3.json"
|
|
data = json.loads(path.read_text(encoding="utf-8"))
|
|
assert data["epoch"] == 3
|
|
assert data["system_tool_action"] == "updated"
|
|
assert data["momentum_updated_task_types"] == ["temporal", "causal"]
|
|
assert data["best_val_acc"] == pytest.approx(0.88)
|