feat(harness): HarnessLog SQLite wrapper + RunLogImpl readonly port
- HarnessLog: TRM4 direct port with WAL mode, threading.Lock, INSERT OR IGNORE idempotent _runs, context manager (completed/failed), create_table with auto run_id+timestamp, insert/insert_many/execute/query/log_event - RunLogImpl: implements core/evolution/protocols.py::RunLog Protocol with independent sqlite3.connect for read-only SELECT (no _runs pollution), asyncio.to_thread wrapping for async interface - _read_table: shared readonly helper with optional question_ids filtering, graceful empty-list return for missing tables - Tests: 17 cases covering thread safety, idempotent inserts, context manager status, WAL mode, protocol compliance, readonly isolation Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,347 @@
|
||||
"""HarnessLog + RunLogImpl 单元测试。
|
||||
|
||||
HarnessLog: SQLite 薄包装的线程安全、幂等、context manager 语义验证。
|
||||
RunLogImpl: 只读 RunLog Protocol 实现，独立连接不污染 _runs 运行状态。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
import threading
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
|
||||
from app.harness.log import HarnessLog, RunLogImpl
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def db_path(tmp_path: Path) -> str:
|
||||
"""返回临时 SQLite 数据库路径。"""
|
||||
return str(tmp_path / "test.db")
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def run_id() -> str:
|
||||
return "test-run-001"
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# HarnessLog 测试
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
class TestHarnessLog:
|
||||
"""HarnessLog 核心功能测试。"""
|
||||
|
||||
def test_create_table_and_insert(self, db_path: str, run_id: str) -> None:
|
||||
"""create_table 建表 + insert 写入 + query 读回。"""
|
||||
with HarnessLog(db_path, run_id) as log:
|
||||
log.create_table("metrics", {"epoch": "INTEGER", "loss": "REAL"})
|
||||
log.insert("metrics", {"epoch": 1, "loss": 0.5})
|
||||
rows = log.query("SELECT * FROM metrics WHERE run_id = ?", (run_id,))
|
||||
|
||||
assert len(rows) == 1
|
||||
assert rows[0]["epoch"] == 1
|
||||
assert rows[0]["loss"] == 0.5
|
||||
assert rows[0]["run_id"] == run_id
|
||||
# insert 自动填充 timestamp
|
||||
assert rows[0]["timestamp"] is not None
|
||||
|
||||
def test_query_thread_safety(self, db_path: str, run_id: str) -> None:
|
||||
"""并发读写不损坏游标状态。"""
|
||||
errors: list[Exception] = []
|
||||
|
||||
with HarnessLog(db_path, run_id) as log:
|
||||
log.create_table("counter", {"value": "INTEGER"})
|
||||
|
||||
def writer() -> None:
|
||||
try:
|
||||
for i in range(50):
|
||||
log.insert("counter", {"value": i})
|
||||
except Exception as exc:
|
||||
errors.append(exc)
|
||||
|
||||
def reader() -> None:
|
||||
try:
|
||||
for _ in range(50):
|
||||
log.query("SELECT COUNT(*) as cnt FROM counter")
|
||||
except Exception as exc:
|
||||
errors.append(exc)
|
||||
|
||||
threads = [
|
||||
threading.Thread(target=writer),
|
||||
threading.Thread(target=reader),
|
||||
threading.Thread(target=reader),
|
||||
]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
|
||||
assert errors == [], f"并发读写产生错误: {errors}"
|
||||
|
||||
def test_context_manager_completed(self, db_path: str, run_id: str) -> None:
|
||||
"""正常退出时 status = completed。"""
|
||||
with HarnessLog(db_path, run_id):
|
||||
pass
|
||||
|
||||
conn = sqlite3.connect(db_path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
row = conn.execute(
|
||||
"SELECT status, finished_at FROM _runs WHERE run_id = ?", (run_id,)
|
||||
).fetchone()
|
||||
conn.close()
|
||||
|
||||
assert row["status"] == "completed"
|
||||
assert row["finished_at"] is not None
|
||||
|
||||
def test_context_manager_failed(self, db_path: str, run_id: str) -> None:
|
||||
"""异常退出时 status = failed。"""
|
||||
with pytest.raises(ValueError, match="boom"), HarnessLog(db_path, run_id):
|
||||
raise ValueError("boom")
|
||||
|
||||
conn = sqlite3.connect(db_path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
row = conn.execute(
|
||||
"SELECT status FROM _runs WHERE run_id = ?", (run_id,)
|
||||
).fetchone()
|
||||
conn.close()
|
||||
|
||||
assert row["status"] == "failed"
|
||||
|
||||
def test_insert_or_ignore_idempotent(self, db_path: str, run_id: str) -> None:
|
||||
"""同一 run_id 多次创建 HarnessLog 不报错（INSERT OR IGNORE 幂等）。"""
|
||||
with HarnessLog(db_path, run_id):
|
||||
pass
|
||||
|
||||
# 再次用同一 run_id 打开——不应抛异常
|
||||
with HarnessLog(db_path, run_id) as log:
|
||||
log.create_table("t", {"x": "INTEGER"})
|
||||
log.insert("t", {"x": 42})
|
||||
|
||||
conn = sqlite3.connect(db_path)
|
||||
count = conn.execute(
|
||||
"SELECT COUNT(*) FROM _runs WHERE run_id = ?", (run_id,)
|
||||
).fetchone()[0]
|
||||
conn.close()
|
||||
|
||||
assert count == 1, "INSERT OR IGNORE 应保证 _runs 只有一行"
|
||||
|
||||
def test_wal_mode(self, db_path: str, run_id: str) -> None:
|
||||
"""连接初始化后 journal_mode 应为 WAL。"""
|
||||
with HarnessLog(db_path, run_id) as log:
|
||||
rows = log.query("PRAGMA journal_mode")
|
||||
|
||||
assert rows[0]["journal_mode"].lower() == "wal"
|
||||
|
||||
def test_insert_many(self, db_path: str, run_id: str) -> None:
|
||||
"""insert_many 批量插入多条记录。"""
|
||||
records = [{"epoch": i, "loss": float(i) * 0.1} for i in range(5)]
|
||||
|
||||
with HarnessLog(db_path, run_id) as log:
|
||||
log.create_table("batch", {"epoch": "INTEGER", "loss": "REAL"})
|
||||
log.insert_many("batch", records)
|
||||
rows = log.query(
|
||||
"SELECT * FROM batch WHERE run_id = ? ORDER BY epoch", (run_id,)
|
||||
)
|
||||
|
||||
assert len(rows) == 5
|
||||
assert [r["epoch"] for r in rows] == [0, 1, 2, 3, 4]
|
||||
|
||||
def test_log_event(self, db_path: str, run_id: str) -> None:
|
||||
"""log_event 向 _events 表写入事件。"""
|
||||
with HarnessLog(db_path, run_id) as log:
|
||||
log.log_event("train_start", {"epoch": 1, "lr": 0.001})
|
||||
log.log_event("train_end", {"epoch": 1, "loss": 0.42})
|
||||
rows = log.query(
|
||||
"SELECT * FROM _events WHERE run_id = ? ORDER BY id", (run_id,)
|
||||
)
|
||||
|
||||
assert len(rows) == 2
|
||||
assert rows[0]["event_type"] == "train_start"
|
||||
assert rows[1]["event_type"] == "train_end"
|
||||
|
||||
def test_execute_raw_sql(self, db_path: str, run_id: str) -> None:
|
||||
"""execute 执行原生 SQL 写操作。"""
|
||||
with HarnessLog(db_path, run_id) as log:
|
||||
log.create_table("raw", {"val": "INTEGER"})
|
||||
log.execute(
|
||||
"INSERT INTO raw (run_id, timestamp, val) VALUES (?, ?, ?)",
|
||||
(run_id, "2026-01-01T00:00:00", 99),
|
||||
)
|
||||
rows = log.query("SELECT val FROM raw WHERE run_id = ?", (run_id,))
|
||||
|
||||
assert len(rows) == 1
|
||||
assert rows[0]["val"] == 99
|
||||
|
||||
def test_upsert_mode(self, db_path: str, run_id: str) -> None:
|
||||
"""insert mode='upsert' 使用 INSERT OR REPLACE。"""
|
||||
with HarnessLog(db_path, run_id) as log:
|
||||
log.create_table("kv", {"key": "TEXT", "val": "TEXT"}, primary_key="key")
|
||||
log.insert("kv", {"key": "a", "val": "1"}, mode="upsert")
|
||||
log.insert("kv", {"key": "a", "val": "2"}, mode="upsert")
|
||||
rows = log.query("SELECT val FROM kv WHERE key = 'a'")
|
||||
|
||||
assert len(rows) == 1
|
||||
assert rows[0]["val"] == "2"
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# RunLogImpl 测试
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
def _setup_predictions_and_traces(db_path: str, run_id: str) -> None:
|
||||
"""向测试数据库写入 predictions 和 traces 数据，模拟 inference 阶段产物。"""
|
||||
with HarnessLog(db_path, run_id) as log:
|
||||
log.create_table(
|
||||
"predictions",
|
||||
{
|
||||
"question_id": "TEXT",
|
||||
"predicted_answer": "TEXT",
|
||||
"correct": "INTEGER",
|
||||
},
|
||||
)
|
||||
log.create_table(
|
||||
"traces",
|
||||
{
|
||||
"question_id": "TEXT",
|
||||
"step_idx": "INTEGER",
|
||||
"action": "TEXT",
|
||||
"observation": "TEXT",
|
||||
},
|
||||
)
|
||||
log.insert_many(
|
||||
"predictions",
|
||||
[
|
||||
{"question_id": "q1", "predicted_answer": "A", "correct": 1},
|
||||
{"question_id": "q2", "predicted_answer": "B", "correct": 0},
|
||||
{"question_id": "q3", "predicted_answer": "C", "correct": 1},
|
||||
],
|
||||
)
|
||||
log.insert_many(
|
||||
"traces",
|
||||
[
|
||||
{
|
||||
"question_id": "q1",
|
||||
"step_idx": 0,
|
||||
"action": "search",
|
||||
"observation": "found",
|
||||
},
|
||||
{
|
||||
"question_id": "q1",
|
||||
"step_idx": 1,
|
||||
"action": "verify",
|
||||
"observation": "ok",
|
||||
},
|
||||
{
|
||||
"question_id": "q2",
|
||||
"step_idx": 0,
|
||||
"action": "search",
|
||||
"observation": "not found",
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
class TestRunLogImpl:
|
||||
"""RunLogImpl 只读查询端口测试。"""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_predictions(self, db_path: str, run_id: str) -> None:
|
||||
"""get_predictions 返回指定 run 的全部预测记录。"""
|
||||
_setup_predictions_and_traces(db_path, run_id)
|
||||
impl = RunLogImpl(db_path)
|
||||
|
||||
preds = await impl.get_predictions(run_id)
|
||||
|
||||
assert len(preds) == 3
|
||||
q_ids = {p["question_id"] for p in preds}
|
||||
assert q_ids == {"q1", "q2", "q3"}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_predictions_filtered(self, db_path: str, run_id: str) -> None:
|
||||
"""get_predictions 按 question_ids 过滤。"""
|
||||
_setup_predictions_and_traces(db_path, run_id)
|
||||
impl = RunLogImpl(db_path)
|
||||
|
||||
preds = await impl.get_predictions(run_id, question_ids=["q1", "q3"])
|
||||
|
||||
assert len(preds) == 2
|
||||
q_ids = {p["question_id"] for p in preds}
|
||||
assert q_ids == {"q1", "q3"}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_traces(self, db_path: str, run_id: str) -> None:
|
||||
"""get_traces 返回指定 run 的全部轨迹记录。"""
|
||||
_setup_predictions_and_traces(db_path, run_id)
|
||||
impl = RunLogImpl(db_path)
|
||||
|
||||
traces = await impl.get_traces(run_id)
|
||||
|
||||
assert len(traces) == 3
|
||||
# q1 有 2 步，q2 有 1 步
|
||||
q1_traces = [t for t in traces if t["question_id"] == "q1"]
|
||||
assert len(q1_traces) == 2
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_traces_filtered(self, db_path: str, run_id: str) -> None:
|
||||
"""get_traces 按 question_ids 过滤。"""
|
||||
_setup_predictions_and_traces(db_path, run_id)
|
||||
impl = RunLogImpl(db_path)
|
||||
|
||||
traces = await impl.get_traces(run_id, question_ids=["q2"])
|
||||
|
||||
assert len(traces) == 1
|
||||
assert traces[0]["question_id"] == "q2"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_readonly_no_runs_insert(self, db_path: str, run_id: str) -> None:
|
||||
"""RunLogImpl 查询不触发 _runs INSERT（不污染运行状态）。"""
|
||||
_setup_predictions_and_traces(db_path, run_id)
|
||||
|
||||
# 用不同 run_id 查询，确保不会创建新的 _runs 行
|
||||
other_run = "nonexistent-run"
|
||||
impl = RunLogImpl(db_path)
|
||||
preds = await impl.get_predictions(other_run)
|
||||
|
||||
assert preds == []
|
||||
|
||||
# 验证 _runs 表没有 other_run 的记录
|
||||
conn = sqlite3.connect(db_path)
|
||||
count = conn.execute(
|
||||
"SELECT COUNT(*) FROM _runs WHERE run_id = ?", (other_run,)
|
||||
).fetchone()[0]
|
||||
conn.close()
|
||||
assert count == 0, "RunLogImpl 不应向 _runs 插入记录"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_protocol_compliance(self, db_path: str, run_id: str) -> None:
|
||||
"""RunLogImpl 满足 RunLog Protocol（runtime_checkable isinstance 检查）。"""
|
||||
from core.evolution.protocols import RunLog
|
||||
|
||||
impl = RunLogImpl(db_path)
|
||||
assert isinstance(impl, RunLog)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_table_returns_empty(self, db_path: str, run_id: str) -> None:
|
||||
"""查询不存在的表（predictions/traces 未建）返回空列表。"""
|
||||
# 仅创建 _runs，不建 predictions/traces 表
|
||||
with HarnessLog(db_path, run_id):
|
||||
pass
|
||||
|
||||
impl = RunLogImpl(db_path)
|
||||
preds = await impl.get_predictions(run_id)
|
||||
traces = await impl.get_traces(run_id)
|
||||
|
||||
assert preds == []
|
||||
assert traces == []
|
||||
Reference in New Issue
Block a user