iomgaa/Video-Tree-TRM5
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add v3 question-gen observation tables to run_store

Browse Source
This commit is contained in:
iomgaa
2026-07-15 09:33:40 -04:00
parent 1ecb6ba25c
commit 5ecbac620c
2 changed files with 770 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/question_gen/run_store.py
+403
View File
@@ -162,6 +162,102 @@ _DDL_VERDICTS_INDEXES = [
] ]
# ---------------------------------------------------------------------------
# v3 出题领域观测表 DDL（facts/unit_verdict/collapse_metrics/quarantine/resume_state）
#
# 时间戳边界（Codex I-4）：v3 新表的 ts 列一律由 insert 方法外部传入，DDL 中不设
# DEFAULT (datetime('now'))、方法体不调 datetime.now()，以保证观测幂等可复现——
# 断点续跑重放时不因进程内时钟漂移产生脏数据。resume_state 无 ts 列（schema 未列）。
# ---------------------------------------------------------------------------
_DDL_FACTS = """
CREATE TABLE IF NOT EXISTS facts (
fact_id TEXT PRIMARY KEY,
video_id TEXT,
segment_id TEXT,
subject TEXT,
action TEXT,
object TEXT,
frame_ids TEXT,
polarity TEXT,
fact_type TEXT,
difficulty_tier INTEGER,
verifier_refs TEXT,
cross_agree INTEGER,
negative_at_target TEXT,
session_id TEXT,
ts TEXT
);
"""
_DDL_UNIT_VERDICT = """
CREATE TABLE IF NOT EXISTS unit_verdict (
unit_id TEXT NOT NULL,
pair_id TEXT,
sub_pattern TEXT,
stage INTEGER NOT NULL,
verdict TEXT,
reason TEXT,
metric_value REAL,
model TEXT,
session_id TEXT,
ts TEXT,
PRIMARY KEY (unit_id, stage)
);
"""
_DDL_COLLAPSE_METRICS = """
CREATE TABLE IF NOT EXISTS collapse_metrics (
pair_id TEXT PRIMARY KEY,
text_only_acc REAL,
single_frame_acc REAL,
placebo_drop REAL,
majority_vote_hit REAL,
slot_chi2 REAL,
distractor_min_dist REAL,
multiformat_consistency REAL,
subtitle_answerability REAL,
ts TEXT
);
"""
_DDL_QUARANTINE = """
CREATE TABLE IF NOT EXISTS quarantine (
content_fingerprint TEXT PRIMARY KEY,
sub_pattern TEXT,
quarantine_reason TEXT,
round_no INTEGER,
ts TEXT
);
"""
_DDL_RESUME_STATE = """
CREATE TABLE IF NOT EXISTS resume_state (
unit_id TEXT PRIMARY KEY,
status TEXT,
config_fingerprint TEXT,
seq_offset INTEGER
);
"""
_DDL_V3_TABLES = [
_DDL_FACTS,
_DDL_UNIT_VERDICT,
_DDL_COLLAPSE_METRICS,
_DDL_QUARANTINE,
_DDL_RESUME_STATE,
]
_DDL_V3_INDEXES = [
"CREATE INDEX IF NOT EXISTS idx_facts_video ON facts(video_id);",
"CREATE INDEX IF NOT EXISTS idx_facts_session ON facts(session_id);",
"CREATE INDEX IF NOT EXISTS idx_uv_unit ON unit_verdict(unit_id);",
"CREATE INDEX IF NOT EXISTS idx_uv_sub_pattern ON unit_verdict(sub_pattern);",
"CREATE INDEX IF NOT EXISTS idx_quar_sub_pattern ON quarantine(sub_pattern);",
"CREATE INDEX IF NOT EXISTS idx_resume_status ON resume_state(status);",
]
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# Store 实现 # Store 实现
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
@@ -201,6 +297,10 @@ class QuestionGenStore:
self._conn.execute(_DDL_VERDICTS) self._conn.execute(_DDL_VERDICTS)
for idx_sql in _DDL_VERDICTS_INDEXES: for idx_sql in _DDL_VERDICTS_INDEXES:
self._conn.execute(idx_sql) self._conn.execute(idx_sql)
for ddl in _DDL_V3_TABLES:
self._conn.execute(ddl)
for idx_sql in _DDL_V3_INDEXES:
self._conn.execute(idx_sql)
self._conn.commit() self._conn.commit()
# 幂等迁移：为已有表加 sub_pattern / selector_scores 列 # 幂等迁移：为已有表加 sub_pattern / selector_scores 列
@@ -648,6 +748,309 @@ class QuestionGenStore:
return {row[0]: "accepted" for row in rows} return {row[0]: "accepted" for row in rows}
# -----------------------------------------------------------------------
# v3 出题领域观测写入（ts 一律外部传入，禁进程内 now，保幂等可复现）
# -----------------------------------------------------------------------
def insert_fact(
self,
*,
fact_id: str,
video_id: str,
segment_id: str,
subject: str,
action: str,
object: str, # noqa: A002 — 与设计列名 object 一致，仅 kwargs 传入无遮蔽风险
frame_ids: str,
polarity: str,
fact_type: str,
difficulty_tier: int,
verifier_refs: str,
cross_agree: int,
negative_at_target: str,
session_id: str,
ts: str,
) -> None:
"""写入一条帧感知抽取 Fact（facts 表，主键 fact_id）。
Parameters
----------
fact_id : str
Fact 唯一标识（UUID）。
video_id, segment_id : str
溯源：所属视频与片段。
subject, action, object : str
结构化绑定三元组。
frame_ids : str
感知所用帧 ID 的 JSON 数组字符串。
polarity : str
事实极性（真/假）。
fact_type : str
事实类型（binding/state/manner/order/evidence）。
difficulty_tier : int
感知难度分层。
verifier_refs : str
双 VLM（qwen/MiniMax）各自裁决的 JSON 字符串。
cross_agree : int
双 VLM 是否一致（0/1）。
negative_at_target : str
目标点为假的核实结果。
session_id : str
epoch/step 关联 ID。
ts : str
观测时间戳 ISO 字符串，由调用方外部传入（禁进程内 now）。
"""
self._conn.execute(
"""
INSERT INTO facts
(fact_id, video_id, segment_id, subject, action, object, frame_ids,
polarity, fact_type, difficulty_tier, verifier_refs, cross_agree,
negative_at_target, session_id, ts)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
fact_id,
video_id,
segment_id,
subject,
action,
object,
frame_ids,
polarity,
fact_type,
difficulty_tier,
verifier_refs,
cross_agree,
negative_at_target,
session_id,
ts,
),
)
self._conn.commit()
logger.debug("facts 已写入: fact_id={}", fact_id)
def insert_unit_verdict(
self,
*,
unit_id: str,
pair_id: str | None,
sub_pattern: str,
stage: int,
verdict: str,
reason: str,
metric_value: float | None,
model: str | None,
session_id: str,
ts: str,
) -> None:
"""写入六层验证某一层的裁决（unit_verdict 表，主键 (unit_id, stage)）。
同一 (unit_id, stage) 重跑时以 upsert 覆盖旧行——每 unit 每层仅保留最新一条，
便于断点续跑重放而不残留过期裁决。
Parameters
----------
unit_id : str
pair/single 单元标识。
pair_id : str | None
所属 pair 关联标识（single 单元可为 None）。
sub_pattern : str
6 子模式之一。
stage : int
验证层号（1-6）。
verdict : str
该层裁决（pass/fail/abstain）。
reason : str
拒因描述。
metric_value : float | None
该层量化值（无量化值时为 None）。
model : str | None
裁判模型名（无模型判定时为 None）。
session_id : str
epoch/step 关联 ID。
ts : str
观测时间戳 ISO 字符串，由调用方外部传入（禁进程内 now）。
"""
self._conn.execute(
"""
INSERT INTO unit_verdict
(unit_id, pair_id, sub_pattern, stage, verdict, reason,
metric_value, model, session_id, ts)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(unit_id, stage) DO UPDATE SET
pair_id=excluded.pair_id,
sub_pattern=excluded.sub_pattern,
verdict=excluded.verdict,
reason=excluded.reason,
metric_value=excluded.metric_value,
model=excluded.model,
session_id=excluded.session_id,
ts=excluded.ts
""",
(
unit_id,
pair_id,
sub_pattern,
stage,
verdict,
reason,
metric_value,
model,
session_id,
ts,
),
)
self._conn.commit()
logger.debug("unit_verdict 已写入: unit_id={}, stage={}", unit_id, stage)
def insert_collapse_metrics(
self,
*,
pair_id: str,
text_only_acc: float,
single_frame_acc: float,
placebo_drop: float,
majority_vote_hit: float,
slot_chi2: float,
distractor_min_dist: float,
multiformat_consistency: float,
subtitle_answerability: float,
ts: str,
) -> None:
"""写入配对坍缩度量（collapse_metrics 表，主键 pair_id）。
同一 pair_id 重算时以 upsert 覆盖旧行——每 pair 仅保留最新一组度量。
Parameters
----------
pair_id : str
配对唯一标识。
text_only_acc, single_frame_acc, placebo_drop : float
模型探针类度量。
majority_vote_hit, slot_chi2, distractor_min_dist : float
纯结构类度量。
multiformat_consistency, subtitle_answerability : float
探针类度量。
ts : str
观测时间戳 ISO 字符串，由调用方外部传入（禁进程内 now）。
"""
self._conn.execute(
"""
INSERT INTO collapse_metrics
(pair_id, text_only_acc, single_frame_acc, placebo_drop,
majority_vote_hit, slot_chi2, distractor_min_dist,
multiformat_consistency, subtitle_answerability, ts)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(pair_id) DO UPDATE SET
text_only_acc=excluded.text_only_acc,
single_frame_acc=excluded.single_frame_acc,
placebo_drop=excluded.placebo_drop,
majority_vote_hit=excluded.majority_vote_hit,
slot_chi2=excluded.slot_chi2,
distractor_min_dist=excluded.distractor_min_dist,
multiformat_consistency=excluded.multiformat_consistency,
subtitle_answerability=excluded.subtitle_answerability,
ts=excluded.ts
""",
(
pair_id,
text_only_acc,
single_frame_acc,
placebo_drop,
majority_vote_hit,
slot_chi2,
distractor_min_dist,
multiformat_consistency,
subtitle_answerability,
ts,
),
)
self._conn.commit()
logger.debug("collapse_metrics 已写入: pair_id={}", pair_id)
def quarantine(
self,
*,
content_fingerprint: str,
sub_pattern: str,
quarantine_reason: str,
round_no: int,
ts: str,
) -> None:
"""将失败题的内容指纹写入隔离区黑名单（quarantine 表，主键 content_fingerprint）。
同一 content_fingerprint 重复调用以 upsert 覆盖——保证语义相同题面只占一行，
补构造前查此表当黑名单去重。
Parameters
----------
content_fingerprint : str
题面语义内容指纹（去重键）。
sub_pattern : str
题目所属子模式。
quarantine_reason : str
隔离原因。
round_no : int
隔离发生的过滤轮次。
ts : str
观测时间戳 ISO 字符串，由调用方外部传入（禁进程内 now）。
"""
self._conn.execute(
"""
INSERT INTO quarantine
(content_fingerprint, sub_pattern, quarantine_reason, round_no, ts)
VALUES (?, ?, ?, ?, ?)
ON CONFLICT(content_fingerprint) DO UPDATE SET
sub_pattern=excluded.sub_pattern,
quarantine_reason=excluded.quarantine_reason,
round_no=excluded.round_no,
ts=excluded.ts
""",
(content_fingerprint, sub_pattern, quarantine_reason, round_no, ts),
)
self._conn.commit()
logger.debug("quarantine 已写入: fingerprint={}", content_fingerprint)
def upsert_resume_state(
self,
*,
unit_id: str,
status: str,
config_fingerprint: str,
seq_offset: int,
) -> None:
"""写入/更新断点续跑状态（resume_state 表，主键 unit_id）。
同一 unit_id 覆盖更新——每单元仅保留最新续跑状态。config_fingerprint 变化时
由调用方据此判定旧进度作废并重跑。本表无 ts 列（schema 未定义），但方法体仍禁
进程内 now，保持 v3 观测一致的幂等可复现语义。
Parameters
----------
unit_id : str
单元唯一标识。
status : str
续跑状态（pending/accepted/rejected）。
config_fingerprint : str
求解器/裁判 config 指纹，变更时旧进度作废。
seq_offset : int
补构造续编偏移，防止编号相撞。
"""
self._conn.execute(
"""
INSERT INTO resume_state
(unit_id, status, config_fingerprint, seq_offset)
VALUES (?, ?, ?, ?)
ON CONFLICT(unit_id) DO UPDATE SET
status=excluded.status,
config_fingerprint=excluded.config_fingerprint,
seq_offset=excluded.seq_offset
""",
(unit_id, status, config_fingerprint, seq_offset),
)
self._conn.commit()
logger.debug("resume_state 已写入: unit_id={}, status={}", unit_id, status)
def close(self) -> None: def close(self) -> None:
"""关闭数据库连接。""" """关闭数据库连接。"""
self._conn.close() self._conn.close()
tests/unit/test_run_store_v3_tables.py
+367
View File
@@ -0,0 +1,367 @@
"""v3 出题领域观测表落库测试 — facts/unit_verdict/collapse_metrics/quarantine/resume_state。
验证 5 张 v3 表的 insert 方法落库+读回、幂等 upsert（同指纹/同 unit_id 不产生重复行）、
以及 ts 由外部传入（禁进程内 now，保幂等可复现）。同时确认 v2 旧表未受本 Task 影响。
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import pytest
from app.question_gen.run_store import QuestionGenStore, RunStats
if TYPE_CHECKING:
from pathlib import Path
@pytest.fixture
def store(tmp_path: Path) -> QuestionGenStore:
"""基于真临时 SQLite 文件构造 QuestionGenStore。"""
db_path = tmp_path / "qgen_v3.db"
s = QuestionGenStore(db_path)
yield s
s.close()
# ---------------------------------------------------------------------------
# 表 1：facts
# ---------------------------------------------------------------------------
def test_insert_fact_roundtrip(store: QuestionGenStore) -> None:
"""insert_fact 落库后可读回全部列，ts 为外部传入值。"""
ts = "2026-07-15T10:00:00+00:00"
store.insert_fact(
fact_id="fact-1",
video_id="vid-1",
segment_id="seg-1",
subject="man",
action="pour",
object="water",
frame_ids="[10, 20, 30]",
polarity="真",
fact_type="binding",
difficulty_tier=2,
verifier_refs='{"qwen": "pass", "minimax": "pass"}',
cross_agree=1,
negative_at_target="not_false",
session_id="sess-1",
ts=ts,
)
row = store._conn.execute(
"SELECT fact_id, video_id, segment_id, subject, action, object, frame_ids, "
"polarity, fact_type, difficulty_tier, verifier_refs, cross_agree, "
"negative_at_target, session_id, ts FROM facts WHERE fact_id='fact-1'"
).fetchone()
assert row == (
"fact-1",
"vid-1",
"seg-1",
"man",
"pour",
"water",
"[10, 20, 30]",
"真",
"binding",
2,
'{"qwen": "pass", "minimax": "pass"}',
1,
"not_false",
"sess-1",
ts,
)
def test_insert_fact_ts_is_external_not_now(store: QuestionGenStore) -> None:
"""ts 如实存储外部传入的历史时间戳，不被进程内 now 覆盖。"""
old_ts = "2020-01-01T00:00:00+00:00"
store.insert_fact(
fact_id="fact-old",
video_id="v",
segment_id="s",
subject="a",
action="b",
object="c",
frame_ids="[]",
polarity="真",
fact_type="state",
difficulty_tier=1,
verifier_refs="{}",
cross_agree=0,
negative_at_target="x",
session_id="sess",
ts=old_ts,
)
stored = store._conn.execute("SELECT ts FROM facts WHERE fact_id='fact-old'").fetchone()[0]
assert stored == old_ts
# ---------------------------------------------------------------------------
# 表 2：unit_verdict
# ---------------------------------------------------------------------------
def test_insert_unit_verdict_roundtrip(store: QuestionGenStore) -> None:
"""insert_unit_verdict 落库后可读回，含可空 pair_id/metric_value/model。"""
ts = "2026-07-15T11:00:00+00:00"
store.insert_unit_verdict(
unit_id="unit-1",
pair_id="pair-1",
sub_pattern="binding_swap",
stage=3,
verdict="pass",
reason="ok",
metric_value=0.87,
model="qwen-vl",
session_id="sess-1",
ts=ts,
)
row = store._conn.execute(
"SELECT unit_id, pair_id, sub_pattern, stage, verdict, reason, metric_value, "
"model, session_id, ts FROM unit_verdict WHERE unit_id='unit-1' AND stage=3"
).fetchone()
assert row == (
"unit-1",
"pair-1",
"binding_swap",
3,
"pass",
"ok",
0.87,
"qwen-vl",
"sess-1",
ts,
)
def test_insert_unit_verdict_nullable_fields(store: QuestionGenStore) -> None:
"""可空列（pair_id/metric_value/model）支持 None 落库。"""
store.insert_unit_verdict(
unit_id="unit-2",
pair_id=None,
sub_pattern="order",
stage=1,
verdict="abstain",
reason="no judge",
metric_value=None,
model=None,
session_id="sess-1",
ts="2026-07-15T11:05:00+00:00",
)
row = store._conn.execute(
"SELECT pair_id, metric_value, model FROM unit_verdict WHERE unit_id='unit-2'"
).fetchone()
assert row == (None, None, None)
def test_insert_unit_verdict_rerun_same_stage_overwrites(store: QuestionGenStore) -> None:
"""同 (unit_id, stage) 重跑覆盖更新，行数保持 1。"""
common = {
"unit_id": "unit-3",
"pair_id": "pair-3",
"sub_pattern": "manner",
"stage": 2,
"session_id": "sess-1",
}
store.insert_unit_verdict(
verdict="fail",
reason="first",
metric_value=0.1,
model="m1",
ts="2026-07-15T11:10:00+00:00",
**common,
)
store.insert_unit_verdict(
verdict="pass",
reason="second",
metric_value=0.9,
model="m2",
ts="2026-07-15T11:20:00+00:00",
**common,
)
rows = store._conn.execute(
"SELECT verdict, reason, metric_value FROM unit_verdict WHERE unit_id='unit-3' AND stage=2"
).fetchall()
assert len(rows) == 1
assert rows[0] == ("pass", "second", 0.9)
# ---------------------------------------------------------------------------
# 表 3：collapse_metrics
# ---------------------------------------------------------------------------
def test_insert_collapse_metrics_roundtrip(store: QuestionGenStore) -> None:
"""insert_collapse_metrics 落库后可读回全部度量列。"""
ts = "2026-07-15T12:00:00+00:00"
store.insert_collapse_metrics(
pair_id="pair-1",
text_only_acc=0.3,
single_frame_acc=0.4,
placebo_drop=0.1,
majority_vote_hit=0.25,
slot_chi2=1.5,
distractor_min_dist=0.6,
multiformat_consistency=0.8,
subtitle_answerability=0.2,
ts=ts,
)
row = store._conn.execute(
"SELECT pair_id, text_only_acc, single_frame_acc, placebo_drop, majority_vote_hit, "
"slot_chi2, distractor_min_dist, multiformat_consistency, subtitle_answerability, ts "
"FROM collapse_metrics WHERE pair_id='pair-1'"
).fetchone()
assert row == (
"pair-1",
0.3,
0.4,
0.1,
0.25,
1.5,
0.6,
0.8,
0.2,
ts,
)
def test_insert_collapse_metrics_recompute_upserts(store: QuestionGenStore) -> None:
"""同 pair_id 重算覆盖更新，行数保持 1。"""
store.insert_collapse_metrics(
pair_id="pair-2",
text_only_acc=0.3,
single_frame_acc=0.4,
placebo_drop=0.1,
majority_vote_hit=0.25,
slot_chi2=1.0,
distractor_min_dist=0.5,
multiformat_consistency=0.7,
subtitle_answerability=0.1,
ts="2026-07-15T12:00:00+00:00",
)
store.insert_collapse_metrics(
pair_id="pair-2",
text_only_acc=0.05,
single_frame_acc=0.05,
placebo_drop=0.5,
majority_vote_hit=0.25,
slot_chi2=9.0,
distractor_min_dist=0.9,
multiformat_consistency=0.95,
subtitle_answerability=0.0,
ts="2026-07-15T12:30:00+00:00",
)
rows = store._conn.execute(
"SELECT text_only_acc, slot_chi2 FROM collapse_metrics WHERE pair_id='pair-2'"
).fetchall()
assert len(rows) == 1
assert rows[0] == (0.05, 9.0)
# ---------------------------------------------------------------------------
# 表 4：quarantine
# ---------------------------------------------------------------------------
def test_quarantine_roundtrip(store: QuestionGenStore) -> None:
"""quarantine 落库后可读回，ts 为外部传入值。"""
ts = "2026-07-15T13:00:00+00:00"
store.quarantine(
content_fingerprint="fp-1",
sub_pattern="binding_swap",
quarantine_reason="double_true",
round_no=1,
ts=ts,
)
row = store._conn.execute(
"SELECT content_fingerprint, sub_pattern, quarantine_reason, round_no, ts "
"FROM quarantine WHERE content_fingerprint='fp-1'"
).fetchone()
assert row == ("fp-1", "binding_swap", "double_true", 1, ts)
def test_quarantine_same_fingerprint_dedup(store: QuestionGenStore) -> None:
"""同 content_fingerprint 重复调用不产生重复行（去重幂等），且内容被更新。"""
store.quarantine(
content_fingerprint="fp-dup",
sub_pattern="order",
quarantine_reason="reason_a",
round_no=1,
ts="2026-07-15T13:00:00+00:00",
)
store.quarantine(
content_fingerprint="fp-dup",
sub_pattern="order",
quarantine_reason="reason_b",
round_no=2,
ts="2026-07-15T13:30:00+00:00",
)
rows = store._conn.execute(
"SELECT quarantine_reason, round_no FROM quarantine WHERE content_fingerprint='fp-dup'"
).fetchall()
assert len(rows) == 1
assert rows[0] == ("reason_b", 2)
# ---------------------------------------------------------------------------
# 表 5：resume_state
# ---------------------------------------------------------------------------
def test_upsert_resume_state_roundtrip(store: QuestionGenStore) -> None:
"""upsert_resume_state 落库后可读回全部列。"""
store.upsert_resume_state(
unit_id="unit-1",
status="pending",
config_fingerprint="cfg-abc",
seq_offset=0,
)
row = store._conn.execute(
"SELECT unit_id, status, config_fingerprint, seq_offset "
"FROM resume_state WHERE unit_id='unit-1'"
).fetchone()
assert row == ("unit-1", "pending", "cfg-abc", 0)
def test_upsert_resume_state_same_unit_overwrites(store: QuestionGenStore) -> None:
"""同 unit_id 覆盖更新 status/config/seq_offset，行数保持 1。"""
store.upsert_resume_state(
unit_id="unit-2",
status="pending",
config_fingerprint="cfg-1",
seq_offset=0,
)
store.upsert_resume_state(
unit_id="unit-2",
status="accepted",
config_fingerprint="cfg-2",
seq_offset=5,
)
rows = store._conn.execute(
"SELECT status, config_fingerprint, seq_offset FROM resume_state WHERE unit_id='unit-2'"
).fetchall()
assert len(rows) == 1
assert rows[0] == ("accepted", "cfg-2", 5)
# ---------------------------------------------------------------------------
# v2 旧表未受影响
# ---------------------------------------------------------------------------
def test_v2_tables_unaffected(store: QuestionGenStore) -> None:
"""v3 建表后 v2 旧表仍存在且旧方法仍可用。"""
store.record_run_start(run_id="run-1", git_sha="deadbeef", config_snapshot="{}")
store.record_run_end(
run_id="run-1",
status="completed",
stats=RunStats(total_slots=10, accepted=8, rejected=2, heavy_sampled=1),
)
stats = store.get_run_stats("run-1")
assert stats.accepted == 8
tables = {
r[0] for r in store._conn.execute("SELECT name FROM sqlite_master WHERE type='table'")
}
assert {"question_gen_runs", "question_gen_items", "adversarial_verdicts"} <= tables