feat: add selector_scores observation column to question_gen_items

This commit is contained in:
2026-07-14 13:59:29 -04:00
parent 3f984acc18
commit 207e834f30
3 changed files with 88 additions and 1 deletions
+28 -1
View File
@@ -125,6 +125,7 @@ CREATE TABLE IF NOT EXISTS question_gen_items (
gate_reject_reason TEXT,
final_status TEXT NOT NULL DEFAULT 'pending',
difficulty_steps INTEGER,
selector_scores TEXT,
created_at TEXT NOT NULL DEFAULT (datetime('now'))
);
"""
@@ -177,11 +178,14 @@ class QuestionGenStore:
self._conn.execute(idx_sql)
self._conn.commit()
# 幂等迁移:为已有表加 sub_pattern 列
# 幂等迁移:为已有表加 sub_pattern / selector_scores
cols = {r[1] for r in self._conn.execute("PRAGMA table_info(question_gen_items)")}
if "sub_pattern" not in cols:
self._conn.execute("ALTER TABLE question_gen_items ADD COLUMN sub_pattern TEXT")
self._conn.commit()
if "selector_scores" not in cols:
self._conn.execute("ALTER TABLE question_gen_items ADD COLUMN selector_scores TEXT")
self._conn.commit()
def record_run_start(self, run_id: str, git_sha: str, config_snapshot: str) -> None:
"""记录批次开始。
@@ -378,6 +382,29 @@ class QuestionGenStore:
if cursor.rowcount == 0:
raise ValueError(f"item_id 不存在: {item_id}")
def update_selector_scores(self, item_id: str, selector_scores_json: str) -> None:
"""写入 grounded selector 打分观测(JSON 字符串)。
Parameters
----------
item_id : str
题目唯一 ID。
selector_scores_json : str
观测 JSONcorrect_score / chosen / pool_size / anneal_rounds / hard_fail。
Raises
------
ValueError
item_id 不存在时抛出。
"""
cursor = self._conn.execute(
"UPDATE question_gen_items SET selector_scores=? WHERE item_id=?",
(selector_scores_json, item_id),
)
self._conn.commit()
if cursor.rowcount == 0:
raise ValueError(f"item_id 不存在: {item_id}")
def get_run_stats(self, run_id: str) -> RunStats:
"""查询批次统计摘要。
@@ -28,8 +28,11 @@ date: 2026-07-12
| `reject_reason` | TEXT | | 拒因文本(首个拒绝门的判定摘要) |
| `final_status` | TEXT | NOT NULL | accepted / rejected / pending |
| `difficulty_steps` | INTEGER | | 重量抽检产出 Agent 步数(NULL=未抽检) |
| `selector_scores` | TEXT | | grounded selector 打分观测(JSON,见下方结构;NULL=未打分) |
| `created_at` | TEXT | NOT NULL | ISO8601 |
`selector_scores` JSON 结构:`{correct_score: float, chosen: float[], pool_size: int, anneal_rounds: int, delta_high_final: float, hard_fail: bool}`
## DDL
```sql
@@ -49,6 +52,7 @@ CREATE TABLE IF NOT EXISTS question_gen_items (
reject_reason TEXT,
final_status TEXT NOT NULL DEFAULT 'pending',
difficulty_steps INTEGER,
selector_scores TEXT,
created_at TEXT NOT NULL
);
@@ -0,0 +1,56 @@
"""selector_scores 列幂等迁移 + update_selector_scores 写入。"""
import json
from app.question_gen.run_store import QuestionGenStore
def _store(tmp_path):
return QuestionGenStore(tmp_path / "q.db")
def test_selector_scores_column_exists(tmp_path):
store = _store(tmp_path)
cols = {r[1] for r in store._conn.execute("PRAGMA table_info(question_gen_items)")}
assert "selector_scores" in cols
store.close()
def test_update_selector_scores_writes_json(tmp_path):
store = _store(tmp_path)
store.record_run_start("run1", "sha", "{}")
store.record_item(
item_id="it1",
run_id="run1",
slot_id="s1",
video_id="v1",
family="ACTION_RECOGNITION",
task_type="Action Recognition",
skill_target="M1_AR",
attempt=1,
question_text="?",
sub_pattern="temporal_reasoning_failure",
)
payload = {
"correct_score": 0.8,
"chosen": [0.7, 0.6, 0.55],
"pool_size": 24,
"anneal_rounds": 0,
"hard_fail": False,
}
store.update_selector_scores("it1", json.dumps(payload))
row = store._conn.execute(
"SELECT selector_scores FROM question_gen_items WHERE item_id='it1'"
).fetchone()
assert json.loads(row[0])["correct_score"] == 0.8
store.close()
def test_update_selector_scores_unknown_item_raises(tmp_path):
store = _store(tmp_path)
try:
store.update_selector_scores("missing", "{}")
raise AssertionError("应抛 ValueError")
except ValueError:
pass
store.close()