feat: add selector_scores observation column to question_gen_items
This commit is contained in:
@@ -125,6 +125,7 @@ CREATE TABLE IF NOT EXISTS question_gen_items (
|
||||
gate_reject_reason TEXT,
|
||||
final_status TEXT NOT NULL DEFAULT 'pending',
|
||||
difficulty_steps INTEGER,
|
||||
selector_scores TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
"""
|
||||
@@ -177,11 +178,14 @@ class QuestionGenStore:
|
||||
self._conn.execute(idx_sql)
|
||||
self._conn.commit()
|
||||
|
||||
# 幂等迁移:为已有表加 sub_pattern 列
|
||||
# 幂等迁移:为已有表加 sub_pattern / selector_scores 列
|
||||
cols = {r[1] for r in self._conn.execute("PRAGMA table_info(question_gen_items)")}
|
||||
if "sub_pattern" not in cols:
|
||||
self._conn.execute("ALTER TABLE question_gen_items ADD COLUMN sub_pattern TEXT")
|
||||
self._conn.commit()
|
||||
if "selector_scores" not in cols:
|
||||
self._conn.execute("ALTER TABLE question_gen_items ADD COLUMN selector_scores TEXT")
|
||||
self._conn.commit()
|
||||
|
||||
def record_run_start(self, run_id: str, git_sha: str, config_snapshot: str) -> None:
|
||||
"""记录批次开始。
|
||||
@@ -378,6 +382,29 @@ class QuestionGenStore:
|
||||
if cursor.rowcount == 0:
|
||||
raise ValueError(f"item_id 不存在: {item_id}")
|
||||
|
||||
def update_selector_scores(self, item_id: str, selector_scores_json: str) -> None:
|
||||
"""写入 grounded selector 打分观测(JSON 字符串)。
|
||||
|
||||
Parameters
|
||||
----------
|
||||
item_id : str
|
||||
题目唯一 ID。
|
||||
selector_scores_json : str
|
||||
观测 JSON:correct_score / chosen / pool_size / anneal_rounds / hard_fail。
|
||||
|
||||
Raises
|
||||
------
|
||||
ValueError
|
||||
item_id 不存在时抛出。
|
||||
"""
|
||||
cursor = self._conn.execute(
|
||||
"UPDATE question_gen_items SET selector_scores=? WHERE item_id=?",
|
||||
(selector_scores_json, item_id),
|
||||
)
|
||||
self._conn.commit()
|
||||
if cursor.rowcount == 0:
|
||||
raise ValueError(f"item_id 不存在: {item_id}")
|
||||
|
||||
def get_run_stats(self, run_id: str) -> RunStats:
|
||||
"""查询批次统计摘要。
|
||||
|
||||
|
||||
@@ -28,8 +28,11 @@ date: 2026-07-12
|
||||
| `reject_reason` | TEXT | | 拒因文本(首个拒绝门的判定摘要) |
|
||||
| `final_status` | TEXT | NOT NULL | accepted / rejected / pending |
|
||||
| `difficulty_steps` | INTEGER | | 重量抽检产出 Agent 步数(NULL=未抽检) |
|
||||
| `selector_scores` | TEXT | | grounded selector 打分观测(JSON,见下方结构;NULL=未打分) |
|
||||
| `created_at` | TEXT | NOT NULL | ISO8601 |
|
||||
|
||||
`selector_scores` JSON 结构:`{correct_score: float, chosen: float[], pool_size: int, anneal_rounds: int, delta_high_final: float, hard_fail: bool}`。
|
||||
|
||||
## DDL
|
||||
|
||||
```sql
|
||||
@@ -49,6 +52,7 @@ CREATE TABLE IF NOT EXISTS question_gen_items (
|
||||
reject_reason TEXT,
|
||||
final_status TEXT NOT NULL DEFAULT 'pending',
|
||||
difficulty_steps INTEGER,
|
||||
selector_scores TEXT,
|
||||
created_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
"""selector_scores 列幂等迁移 + update_selector_scores 写入。"""
|
||||
|
||||
import json
|
||||
|
||||
from app.question_gen.run_store import QuestionGenStore
|
||||
|
||||
|
||||
def _store(tmp_path):
|
||||
return QuestionGenStore(tmp_path / "q.db")
|
||||
|
||||
|
||||
def test_selector_scores_column_exists(tmp_path):
|
||||
store = _store(tmp_path)
|
||||
cols = {r[1] for r in store._conn.execute("PRAGMA table_info(question_gen_items)")}
|
||||
assert "selector_scores" in cols
|
||||
store.close()
|
||||
|
||||
|
||||
def test_update_selector_scores_writes_json(tmp_path):
|
||||
store = _store(tmp_path)
|
||||
store.record_run_start("run1", "sha", "{}")
|
||||
store.record_item(
|
||||
item_id="it1",
|
||||
run_id="run1",
|
||||
slot_id="s1",
|
||||
video_id="v1",
|
||||
family="ACTION_RECOGNITION",
|
||||
task_type="Action Recognition",
|
||||
skill_target="M1_AR",
|
||||
attempt=1,
|
||||
question_text="?",
|
||||
sub_pattern="temporal_reasoning_failure",
|
||||
)
|
||||
payload = {
|
||||
"correct_score": 0.8,
|
||||
"chosen": [0.7, 0.6, 0.55],
|
||||
"pool_size": 24,
|
||||
"anneal_rounds": 0,
|
||||
"hard_fail": False,
|
||||
}
|
||||
store.update_selector_scores("it1", json.dumps(payload))
|
||||
row = store._conn.execute(
|
||||
"SELECT selector_scores FROM question_gen_items WHERE item_id='it1'"
|
||||
).fetchone()
|
||||
assert json.loads(row[0])["correct_score"] == 0.8
|
||||
store.close()
|
||||
|
||||
|
||||
def test_update_selector_scores_unknown_item_raises(tmp_path):
|
||||
store = _store(tmp_path)
|
||||
try:
|
||||
store.update_selector_scores("missing", "{}")
|
||||
raise AssertionError("应抛 ValueError")
|
||||
except ValueError:
|
||||
pass
|
||||
store.close()
|
||||
Reference in New Issue
Block a user