diff --git a/research-wiki/graph/edges.json b/research-wiki/graph/edges.json index 39b3084..1b5a04e 100644 --- a/research-wiki/graph/edges.json +++ b/research-wiki/graph/edges.json @@ -110,6 +110,41 @@ "id": "plan:batch-tree-build-plan", "label": "Spec-2 建树批量并行实现计划", "type": "plan" + }, + { + "id": "schema:question-gen-runs", + "label": "表结构: question_gen_runs（出题批次）", + "type": "schema" + }, + { + "id": "schema:question-gen-items", + "label": "表结构: question_gen_items（逐题门判定）", + "type": "schema" + }, + { + "id": "metric:qgen-gate-rejection-rate", + "label": "出题四门总拦截率", + "type": "metric" + }, + { + "id": "metric:qgen-regen-convergence", + "label": "重出收敛率（3轮内过门）", + "type": "metric" + }, + { + "id": "metric:qgen-acceptance-rate", + "label": "最终接受率", + "type": "metric" + }, + { + "id": "metric:qgen-answer-uniformity", + "label": "答案位置分布均匀性", + "type": "metric" + }, + { + "id": "plan:2026-07-11-question-gen-v2", + "label": "出题管线 v2 实现计划", + "type": "plan" } ], "links": [ @@ -182,6 +217,48 @@ "relation": "implements", "evidence": "实现 Spec-2 四任务", "added": "2026-07-11T15:07:06.178664+00:00" + }, + { + "source": "metric:qgen-gate-rejection-rate", + "target": "schema:question-gen-items", + "relation": "measures", + "evidence": "从 question_gen_items 表 attempt=1 行聚合计算", + "added": "2026-07-12T02:34:28.728290+00:00" + }, + { + "source": "metric:qgen-regen-convergence", + "target": "schema:question-gen-items", + "relation": "measures", + "evidence": "按 slot_id 聚合 final_status 计算收敛率", + "added": "2026-07-12T02:34:29.858363+00:00" + }, + { + "source": "metric:qgen-acceptance-rate", + "target": "schema:question-gen-runs", + "relation": "measures", + "evidence": "直接读取 runs 表 accepted/total_slots", + "added": "2026-07-12T02:34:30.989357+00:00" + }, + { + "source": "schema:question-gen-runs", + "target": "design:2026-07-11-question-gen-v2", + "relation": "implements", + "evidence": "设计 §5 运行时数据要求", + "added": "2026-07-12T02:34:32.121484+00:00" + }, + { + "source": "schema:question-gen-items", + "target": "design:2026-07-11-question-gen-v2", + "relation": "implements", + "evidence": "设计 §5 逐题可追溯要求", + "added": "2026-07-12T02:34:33.267570+00:00" + }, + { + "source": "plan:2026-07-11-question-gen-v2", + "target": "design:2026-07-11-question-gen-v2", + "relation": "implements", + "evidence": "Spec-3 设计的实现计划", + "added": "2026-07-12T02:57:39.569968+00:00" } ] } \ No newline at end of file diff --git a/research-wiki/index.md b/research-wiki/index.md index 05bf1db..112b7d1 100644 --- a/research-wiki/index.md +++ b/research-wiki/index.md @@ -1,6 +1,6 @@ # Research Wiki 索引 -> 自动生成，更新时间：2026-07-11 15:07 UTC +> 自动生成，更新时间：2026-07-12 02:57 UTC ## design (21) - [2026-07-06-core-agent-adapters-llm-design](designs/2026-07-06-core-agent-adapters-llm-design.md) `design:2026-07-06-core-agent-adapters-llm-design` @@ -25,12 +25,13 @@ - [赛题生成工具设计](designs/question-gen-synth.md) `design:question-gen-synth` - [赛题生成工具设计（Question Generation Synthesis）](designs/2026-07-09-question-gen-synth-design.md) `design:2026-07-09-question-gen-synth-design` -## finding (3) +## finding (4) - [2026-07-11-benchmark-failure-taxonomy](findings/2026-07-11-benchmark-failure-taxonomy.md) `finding:2026-07-11-benchmark-failure-taxonomy` - [2026-07-11-question-gen-calibration-analysis](findings/2026-07-11-question-gen-calibration-analysis.md) `finding:2026-07-11-question-gen-calibration-analysis` - [Harness 评估: Spec-1 修复验证 (infer_spec1check)](findings/eval-spec1check.md) `finding:eval-spec1check` +- [Harness 评估: Spec-2 批量并行建树](findings/eval-spec2-batch-tree-build.md) `finding:eval-spec2-batch-tree-build` -## plan (21) +## plan (22) - [2026-07-06-core-agent-adapters-llm](plans/2026-07-06-core-agent-adapters-llm.md) `plan:2026-07-06-core-agent-adapters-llm` - [2026-07-07-app-harness](plans/2026-07-07-app-harness.md) `plan:2026-07-07-app-harness` - [2026-07-07-core-evolution](plans/2026-07-07-core-evolution.md) `plan:2026-07-07-core-evolution` @@ -48,7 +49,18 @@ - [question_gen 模块实现计划](plans/question-gen.md) `plan:question-gen` - [Spec-1 Agent 执行环境修复实现计划](plans/agent-runtime-fixes-plan.md) `plan:agent-runtime-fixes-plan` - [Spec-2 建树批量并行实现计划](plans/batch-tree-build-plan.md) `plan:batch-tree-build-plan` +- [出题管线 v2 实现计划](plans/2026-07-11-question-gen-v2.md) `plan:2026-07-11-question-gen-v2` - [建树修复管线三项改造实现计划](plans/tree-repair-resilience.md) `plan:tree-repair-resilience` - [建树模块竖切实现计划](plans/tree-module-vertical-slice.md) `plan:tree-module-vertical-slice` - [赛题生成工具实现计划](plans/question-gen-synth.md) `plan:question-gen-synth` - [项目基础设施初始化计划](plans/infrastructure-setup.md) `plan:infrastructure-setup` + +## schema (2) +- [表结构: question_gen_items（逐题门判定）](schemas/question-gen-items.md) `schema:question-gen-items` +- [表结构: question_gen_runs（出题批次）](schemas/question-gen-runs.md) `schema:question-gen-runs` + +## metric (4) +- [出题四门总拦截率](metrics/qgen-gate-rejection-rate.md) `metric:qgen-gate-rejection-rate` +- [最终接受率](metrics/qgen-acceptance-rate.md) `metric:qgen-acceptance-rate` +- [答案位置分布均匀性](metrics/qgen-answer-uniformity.md) `metric:qgen-answer-uniformity` +- [重出收敛率（3轮内过门）](metrics/qgen-regen-convergence.md) `metric:qgen-regen-convergence` diff --git a/research-wiki/log.md b/research-wiki/log.md index a874aa6..a2567b6 100644 --- a/research-wiki/log.md +++ b/research-wiki/log.md @@ -50,3 +50,18 @@ - [2026-07-11 15:07 UTC] 新增 plan: Spec-2 建树批量并行实现计划 (plan:batch-tree-build-plan) - [2026-07-11 15:07 UTC] 新增边: plan:batch-tree-build-plan --implements--> design:batch-tree-build - [2026-07-11 15:07 UTC] 重建索引: 45 篇页面 +- [2026-07-12 02:32 UTC] 新增 schema: 表结构: question_gen_runs（出题批次） (schema:question-gen-runs) +- [2026-07-12 02:32 UTC] 新增 schema: 表结构: question_gen_items（逐题门判定） (schema:question-gen-items) +- [2026-07-12 02:33 UTC] 新增 metric: 出题四门总拦截率 (metric:qgen-gate-rejection-rate) +- [2026-07-12 02:33 UTC] 新增 metric: 重出收敛率（3轮内过门） (metric:qgen-regen-convergence) +- [2026-07-12 02:33 UTC] 新增 metric: 最终接受率 (metric:qgen-acceptance-rate) +- [2026-07-12 02:33 UTC] 新增 metric: 答案位置分布均匀性 (metric:qgen-answer-uniformity) +- [2026-07-12 02:34 UTC] 新增边: metric:qgen-gate-rejection-rate --measures--> schema:question-gen-items +- [2026-07-12 02:34 UTC] 新增边: metric:qgen-regen-convergence --measures--> schema:question-gen-items +- [2026-07-12 02:34 UTC] 新增边: metric:qgen-acceptance-rate --measures--> schema:question-gen-runs +- [2026-07-12 02:34 UTC] 新增边: schema:question-gen-runs --implements--> design:2026-07-11-question-gen-v2 +- [2026-07-12 02:34 UTC] 新增边: schema:question-gen-items --implements--> design:2026-07-11-question-gen-v2 +- [2026-07-12 02:34 UTC] 重建索引: 52 篇页面 +- [2026-07-12 02:38 UTC] 新增 plan: 出题管线 v2 实现计划 (plan:2026-07-11-question-gen-v2) +- [2026-07-12 02:57 UTC] 新增边: plan:2026-07-11-question-gen-v2 --implements--> design:2026-07-11-question-gen-v2 +- [2026-07-12 02:57 UTC] 重建索引: 53 篇页面 diff --git a/research-wiki/metrics/qgen-acceptance-rate.md b/research-wiki/metrics/qgen-acceptance-rate.md new file mode 100644 index 0000000..6359aef --- /dev/null +++ b/research-wiki/metrics/qgen-acceptance-rate.md @@ -0,0 +1,32 @@ +--- +type: metric +node_id: metric:qgen-acceptance-rate +title: 最终接受率 +date: 2026-07-12 +--- + +# 最终接受率 + +## 定义 + +最终过门题数 / 目标题数（total_slots）。衡量管线整体产能。 + +## 判定方式 + +- **类型**: 硬性（下限约束） +- **阈值**: ≥ 85%（低于说明管线产能不足，需扩大候选池或降低门阈值） +- **计算**: `accepted / total_slots` +- **数据源**: `question_gen_runs` + +## 基线 + +待首次运行后建立。 + +## SQL + +```sql +SELECT + CAST(accepted AS REAL) / total_slots AS acceptance_rate +FROM question_gen_runs +WHERE run_id=?; +``` diff --git a/research-wiki/metrics/qgen-answer-uniformity.md b/research-wiki/metrics/qgen-answer-uniformity.md new file mode 100644 index 0000000..fb7edeb --- /dev/null +++ b/research-wiki/metrics/qgen-answer-uniformity.md @@ -0,0 +1,33 @@ +--- +type: metric +node_id: metric:qgen-answer-uniformity +title: 答案位置分布均匀性 +date: 2026-07-12 +--- + +# 答案位置分布均匀性 + +## 定义 + +最终接受题目中，正确答案在 A/B/C/D 四个位置的分布是否均匀。 + +## 判定方式 + +- **类型**: 硬性（卡方检验） +- **阈值**: χ² 检验 p > 0.05（不拒绝均匀分布假设） +- **计算**: 对 accepted 题目统计 correct_answer 的 A/B/C/D 频次，做 χ²(df=3) 检验 +- **数据源**: 题目 JSON 文件（correct_answer 字段） + +## 基线 + +设计 §7.3 要求 shuffle 后必须均匀；不依赖历史基线。 + +## 验证方式 + +```python +from scipy.stats import chisquare +observed = [count_A, count_B, count_C, count_D] +expected = [total/4] * 4 +stat, p = chisquare(observed, expected) +assert p > 0.05, f"答案位置分布不均匀: p={p:.4f}" +``` diff --git a/research-wiki/metrics/qgen-gate-rejection-rate.md b/research-wiki/metrics/qgen-gate-rejection-rate.md new file mode 100644 index 0000000..52b477a --- /dev/null +++ b/research-wiki/metrics/qgen-gate-rejection-rate.md @@ -0,0 +1,33 @@ +--- +type: metric +node_id: metric:qgen-gate-rejection-rate +title: 出题四门总拦截率 +date: 2026-07-12 +--- + +# 出题四门总拦截率 + +## 定义 + +轻量四门（键验证 + 盲答 + 多真 + 泄漏）在首次尝试中拒绝的题目占比。 + +## 判定方式 + +- **类型**: 硬性（上限约束） +- **阈值**: ≤ 60%（超过说明生成器质量过差，prompt 模板需大幅迭代） +- **计算**: `COUNT(attempt=1 AND final_status='rejected') / COUNT(attempt=1)` +- **数据源**: `question_gen_items` WHERE `attempt=1` + +## 基线 + +待首次运行后建立。 + +## SQL + +```sql +SELECT + CAST(SUM(CASE WHEN final_status='rejected' THEN 1 ELSE 0 END) AS REAL) / + COUNT(*) AS rejection_rate +FROM question_gen_items +WHERE run_id=? AND attempt=1; +``` diff --git a/research-wiki/metrics/qgen-regen-convergence.md b/research-wiki/metrics/qgen-regen-convergence.md new file mode 100644 index 0000000..0d82eec --- /dev/null +++ b/research-wiki/metrics/qgen-regen-convergence.md @@ -0,0 +1,33 @@ +--- +type: metric +node_id: metric:qgen-regen-convergence +title: 重出收敛率（3轮内过门） +date: 2026-07-12 +--- + +# 重出收敛率（3轮内过门） + +## 定义 + +所有 slot 中，在 ≤3 次尝试内最终被接受的 slot 占总 slot 的比例。 + +## 判定方式 + +- **类型**: 硬性（下限约束） +- **阈值**: ≥ 80%（低于说明 prompt 模板或拒因反馈机制需迭代） +- **计算**: `COUNT(DISTINCT slot_id WHERE accepted) / COUNT(DISTINCT slot_id)` +- **数据源**: `question_gen_items` + +## 基线 + +待首次运行后建立。 + +## SQL + +```sql +SELECT + CAST(COUNT(DISTINCT CASE WHEN final_status='accepted' THEN slot_id END) AS REAL) / + COUNT(DISTINCT slot_id) AS convergence_rate +FROM question_gen_items +WHERE run_id=?; +``` diff --git a/research-wiki/plans/2026-07-11-question-gen-v2.md b/research-wiki/plans/2026-07-11-question-gen-v2.md new file mode 100644 index 0000000..fe7538b --- /dev/null +++ b/research-wiki/plans/2026-07-11-question-gen-v2.md @@ -0,0 +1,571 @@ +--- +type: plan +node_id: plan:2026-07-11-question-gen-v2 +title: "出题管线 v2 实现计划" +date: 2026-07-12 +--- + +# 出题管线 v2 实现计划 + +> **For agentic workers:** REQUIRED SUB-SKILL: Use subagent-driven-development to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** 将出题管线从"按 task_type 平铺生成"升级为"按失败机理题族生成 + 逐题轻量四门质量关 + 重出循环 + 重量抽检"，在不影响 harness/推理侧的前提下显著提升题目区分度与泄漏控制。 + +**Architecture:** 新增 6 个模块（postprocess / families / sampler_v2 / generator_v2 / gates / pipeline_v2）+ 1 个日志存储（run_store），均位于 `app/question_gen/` 下；扩展 `core/types.py` 和 `config/default.yaml`；改造 `tools/generate_questions.py` CLI。v1 `generate_one` 保留不删。 + +**Tech Stack:** Python 3.11, asyncio, loguru, SQLite, GovernedLLMClient/GovernedVLMClient (DI), numpy, pytest + +--- + +## 依赖拓扑 + +```mermaid +graph TD + T1[Task 1: 类型扩展+配置] --> T2[Task 2: 后处理层] + T1 --> T3[Task 3: 题族规格] + T3 --> T4[Task 4: v2 采样器] + T4 --> T5[Task 5: v2 生成器] + T3 --> T6[Task 6: 轻量四门] + T2 --> T6 + T1 --> T7[Task 7: SQLite 日志] + T6 --> T7 + T5 --> T8[Task 8: 编排管线] + T6 --> T8 + T7 --> T8 + T8 --> T9[Task 9: CLI + 回归] +``` + +可并行: T2 / T3 / T7（仅依赖 T1）。 + +--- + +## Task 1: GeneratedQuestion 类型扩展 + 配置 YAML + +**Files:** `core/types.py`(改) | `config/default.yaml`(改) | `app/question_gen/loader.py`(改) | `tests/unit/test_types_v2.py`(新) + +> **Codex 审查补充:** 若 `app/harness/pools.py` 存在 `_q_to_dict`/`_dict_to_q` 转换函数，需同步保留 `skill_target`/`difficulty_steps` 字段。实现者须 grep 确认并修改。 + +- [ ] **Step 1 (RED):** 测试新字段默认 None + 显式赋值 + frozen 不可变 + +```python +# tests/unit/test_types_v2.py +class TestGeneratedQuestionV2: + def test_new_fields_default_none(self): + q = GeneratedQuestion(question_id="q1", video_id="v1", task_type="TN", + question="?", options=("A","B","C","D"), answer="A", source_nodes=("n1",), difficulty="medium") + assert q.skill_target is None and q.difficulty_steps is None + + def test_new_fields_explicit(self): + q = GeneratedQuestion(..., skill_target="M1", difficulty_steps=5) + assert q.skill_target == "M1" + +class TestLoaderV2Compat: + def test_load_old_format_json(self, tmp_path): ... # 无新字段 -> None + def test_load_new_format_json(self, tmp_path): ... # 有新字段 -> 正确加载 +``` + +- [ ] **Step 2 (GREEN):** `core/types.py` 添加 `skill_target: str | None = field(default=None)` 和 `difficulty_steps: int | None = field(default=None)` + +- [ ] **Step 3 (GREEN):** `loader.py` 构造处加 `skill_target=item.get("skill_target"), difficulty_steps=item.get("difficulty_steps")` + +- [ ] **Step 4:** `config/default.yaml` 新增 `question_gen_v2:` 节 + +```yaml +question_gen_v2: + family_ratios: {retrieval: 0.30, reasoning: 0.25, enumeration: 0.20, visual: 0.15, spatial: 0.10} + gate: {blind_answer_model: "gpt-4.1-mini", leak_test_model: "gpt-4.1-mini", + key_verify_model: "gpt-4.1-mini", multi_true_model: "gpt-4.1-mini"} + dedup_threshold: 0.85 + retry_limit: 3 + heavy_sample_rate: 0.15 + heavy_agent_model: "gpt-4.1-mini" + output_dir: "store/questions/generated-v2" + per_type: 20 # 12 类 × 20 = 240 题（设计 §3 硬约束） + concurrency: 4 + seed: 42 +``` + +- [ ] **Step 5 (COMMIT):** `feat(types): extend GeneratedQuestion with skill_target & difficulty_steps` + +--- + +## Task 2: 确定性后处理层 `app/question_gen/postprocess.py` + +**Files:** `app/question_gen/postprocess.py`(新) | `tests/unit/test_postprocess.py`(新) + +### 签名 + +```python +@dataclass(frozen=True) +class PostprocessResult: + options: tuple[str, ...] + answer: str + referent_violations: list[str] + verbatim_ratio: float + has_time_anchor: bool + +def shuffle_options(options: tuple[str, ...], answer: str, rng: random.Random) -> tuple[tuple[str, ...], str]: ... +def check_referent_blacklist(question_text: str) -> list[str]: ... +def check_verbatim(question_text: str, correct_option: str, source_texts: list[str], window: int = 6) -> float: ... +def has_time_anchor(question_text: str) -> bool: ... +def check_forbidden_material(source_nodes_text: str, task_type: str) -> list[str]: ... + """出题禁区：检测 T1 类素材（瞬时动作/记分牌时序/无对白因果）和 T7 噪声模式（选项重复/计数边界口径含糊）。返回违规描述列表，空列表=通过。""" +def run_postprocess(question_text: str, options: tuple[str, ...], answer: str, source_texts: list[str], rng: random.Random) -> PostprocessResult: ... +``` + +- [ ] **Step 1 (RED):** 测试 + +```python +class TestShuffleOptions: + def test_deterministic_with_seed(self): ... + def test_answer_remapped_correctly(self): ... # answer 字母始终指向原正确文本 + +class TestReferentBlacklist: + def test_clean_passes(self): ... + def test_this_clip_caught(self): ... + +class TestVerbatim: + def test_zero_overlap(self): ... + def test_full_copy_returns_one(self): ... + def test_partial(self): ... + +class TestTimeAnchor: + def test_timestamp(self): ... # "at 01:30" -> True + def test_no_anchor(self): ... # -> False +``` + +- [ ] **Step 2 (GREEN):** 实现。逻辑: shuffle 用 index permutation+remap; blacklist 用预编译 `_BLACKLIST_PATTERNS`; verbatim 用 word-level n-gram set intersection ratio; time_anchor 用正则 `\d{1,2}:\d{2}` + 短语列表。 + +- [ ] **Step 3 (COMMIT):** `feat(question_gen): add deterministic postprocess layer` + +--- + +## Task 3: 题族规格声明 `app/question_gen/families.py` + +**Files:** `app/question_gen/families.py`(新) | `tests/unit/test_families.py`(新) + +### 签名 + +```python +@dataclass(frozen=True) +class LeakTestProfile: + shortcut_type: str # "temporal_proximity"/"option_length"/"frequency"/"visual_salience"/"spatial_default" + probe_template: str # store/prompts/question_gen/ 下模板名 + pass_threshold: float + +@dataclass(frozen=True) +class SamplingConstraint: + min_subtitles: int; min_l3_nodes: int; require_frames: bool; cross_l2_span: bool + +@dataclass(frozen=True) +class QuestionFamilySpec: + name: str; skill_target: str; sampling: SamplingConstraint + legal_task_types: frozenset[str]; leak_profile: LeakTestProfile; prompt_template: str + +RETRIEVAL_FAMILY: QuestionFamilySpec # M1, 30% +REASONING_FAMILY: QuestionFamilySpec # M2, 25% +ENUMERATION_FAMILY: QuestionFamilySpec # M3, 20% +VISUAL_FAMILY: QuestionFamilySpec # M4, 15% +SPATIAL_FAMILY: QuestionFamilySpec # M5, 10% +ALL_FAMILIES: tuple[QuestionFamilySpec, ...] + +def get_family_for_slot(task_type: str, family_ratios: dict[str, float], rng: random.Random) -> QuestionFamilySpec: ... +``` + +- [ ] **Step 1 (RED):** + +```python +class TestFamilySpec: + def test_all_families_cover_all_task_types(self): ... # 12 类全覆盖 + def test_skill_targets_unique(self): ... +class TestGetFamilyForSlot: + def test_respects_legal_task_types(self): ... # 返回族必含给定 type + def test_deterministic_with_seed(self): ... + def test_invalid_task_type_raises(self): ... + def test_distribution_approximates_ratios(self): ... # chi-square p>0.01 +``` + +- [ ] **Step 2 (GREEN):** 实现。`get_family_for_slot`: 过滤 legal -> 归一化权重 -> `rng.choices`。 + +- [ ] **Step 3 (COMMIT):** `feat(question_gen): add 5 question family specs with sampling constraints` + +--- + +## Task 4: v2 采样器 `app/question_gen/sampler_v2.py` + +**Files:** `app/question_gen/sampler_v2.py`(新) | `tests/unit/test_sampler_v2.py`(新) + +### 签名 + +```python +@dataclass(frozen=True) +class MaterialContext: + anchor: AnchorContext + source_nodes: tuple[str, ...] + subtitle_sentences: list[str] + frame_paths: list[str] + cross_l2_texts: list[str] + +def _validate_sampling_constraints(tree: TreeIndex, node_id: str, constraint: SamplingConstraint) -> bool: ... +def _collect_subtitle_sentences(tree: TreeIndex, node_ids: tuple[str, ...]) -> list[str]: ... +def _collect_cross_l2_context(tree: TreeIndex, anchor_l2_id: str, max_peers: int = 3) -> list[str]: ... + +def sample_material_v2( + tree: TreeIndex, family_spec: QuestionFamilySpec, task_type: str, + used_node_ids: set[str], rng: random.Random, *, max_attempts: int = 10, +) -> MaterialContext: ... + # Raises RuntimeError if max_attempts exhausted +``` + +- [ ] **Step 1 (RED):** + +```python +class TestSampleMaterialV2: + def test_returns_material_context(self, real_tree): ... + def test_respects_used_nodes(self, real_tree): ... + def test_constraint_violation_retries(self, real_tree): ... # -> RuntimeError + def test_cross_l2_populated_for_reasoning(self, real_tree): ... + def test_subtitle_sentences_from_anchor(self, real_tree): ... +``` + +- [ ] **Step 2 (GREEN):** 实现。路由 `TASK_TYPE_LEVEL_MAP[task_type].level` -> `_sample_lX`; 验证 constraints; 收集 subtitles + cross_l2。 + +- [ ] **Step 3 (COMMIT):** `feat(question_gen): add v2 material sampler with family constraints` + +--- + +## Task 5: v2 生成器 `app/question_gen/generator_v2.py` + +**Files:** `app/question_gen/generator_v2.py`(新) | `store/prompts/question_gen/{retrieval,reasoning,enumeration,visual,spatial}.md`(新) | `tests/unit/test_generator_v2.py`(新) + +### 签名 + +```python +@dataclass(frozen=True) +class CandidateQuestion: + question_id: str; video_id: str; task_type: str; skill_target: str + question: str; options: tuple[str, ...]; answer: str + source_nodes: tuple[str, ...]; difficulty: str + subtitle_sentences: tuple[str, ...] = field(default_factory=tuple) # 验证材料 + frame_paths: tuple[str, ...] = field(default_factory=tuple) + +def _load_prompt_template(family_spec: QuestionFamilySpec) -> str: ... +def _build_v2_prompt(family_spec: QuestionFamilySpec, material: MaterialContext, task_type: str, seq: int, *, reject_reason: str | None = None) -> tuple[list[dict[str, str]], list[str]]: ... +def _parse_v2_response(raw: str, video_id: str, task_type: str, skill_target: str, seq: int, source_nodes: tuple[str, ...]) -> CandidateQuestion: ... + +async def generate_one_v2( + vlm: VLMProvider, tree: TreeIndex, material: MaterialContext, + family_spec: QuestionFamilySpec, task_type: str, seq: int, *, + video_id: str, reject_reason: str | None = None, session_id: str, +) -> CandidateQuestion: ... +``` + +- [ ] **Step 1 (RED):** + +```python +class TestBuildV2Prompt: + def test_includes_family_template(self): ... + def test_reject_reason_injected(self): ... +class TestParseV2Response: + def test_valid_json(self): ... + def test_missing_field_raises(self): ... + def test_invalid_answer_raises(self): ... +class TestGenerateOneV2: + async def test_happy_path(self, mock_vlm): ... # -> CandidateQuestion +``` + +- [ ] **Step 2 (GREEN):** 实现 + 5 个 per-family prompt 模板（JSON output format）。 + +- [ ] **Step 3 (COMMIT):** `feat(question_gen): add v2 generator with per-family prompt templates` + +--- + +## Task 6: 轻量四门 `app/question_gen/gates.py` + +**Files:** `app/question_gen/gates.py`(新) | `store/prompts/question_gen/gate_{key_verify,blind_answer,multi_true,leak_*}.md`(新) | `tests/unit/test_gates.py`(新) + +### 签名 + +```python +class GateVerdict(Enum): PASS = "pass"; FAIL = "fail"; SKIP = "skip" + +@dataclass(frozen=True) +class GateResult: + verdict: GateVerdict; reason: str; raw_response: str + +@dataclass(frozen=True) +class GateReport: + key_verify: GateResult; blind_answer: GateResult; multi_true: GateResult; leak_test: GateResult + @property + def passed(self) -> bool: ... # 全门 PASS|SKIP + @property + def reject_reason(self) -> str | None: ... # 首个 FAIL 门 reason + +async def _gate_key_verify(candidate, tree, llm, *, session_id) -> GateResult: ... +async def _gate_blind_answer(candidate, llm, *, session_id) -> GateResult: ... +async def _gate_multi_true(candidate, tree, llm, *, session_id) -> GateResult: ... +async def _gate_leak_test(candidate, family_spec, llm, *, session_id) -> GateResult: ... + +async def run_gates( + candidate: CandidateQuestion, tree: TreeIndex, llm: LLMProvider, + family_spec: QuestionFamilySpec, postprocess: PostprocessResult, *, session_id: str, +) -> GateReport: ... + # 前置: verbatim_ratio > 0.5 直接 FAIL key_verify; 并发 asyncio.gather 四门 +``` + +- [ ] **Step 1 (RED):** + +```python +class TestGateKeyVerify: + async def test_pass_evidence(self, mock_llm): ... + async def test_fail_no_evidence(self, mock_llm): ... +class TestGateBlindAnswer: + async def test_pass_wrong(self, mock_llm): ... + async def test_fail_correct(self, mock_llm): ... +class TestGateMultiTrue: + async def test_pass_single(self, mock_llm): ... + async def test_fail_multi(self, mock_llm): ... +class TestGateLeakTest: + async def test_per_family_template(self, mock_llm): ... +class TestRunGates: + async def test_all_pass(self): ... + async def test_high_verbatim_shortcircuits(self): ... +``` + +- [ ] **Step 2 (GREEN):** 实现 + 8 个 gate prompt 模板（key_verify / blind_answer / multi_true / leak x 5 族）。 + +- [ ] **Step 3 (COMMIT):** `feat(question_gen): add lightweight 4-gate quality check` + +--- + +## Task 7: SQLite 日志记录器 `app/question_gen/run_store.py` + +**Files:** `app/question_gen/run_store.py`(新) | `tests/unit/test_run_store.py`(新) + +### 签名 + +```python +@dataclass(frozen=True) +class RunStats: + total_slots: int; accepted: int; rejected: int; heavy_sampled: int + +class QuestionGenStore: + def __init__(self, db_path: Path) -> None: ... + def _init_schema(self) -> None: ... # 幂等 DDL（参照 research-wiki/schemas/） + def record_run_start(self, run_id: str, git_sha: str, config_snapshot: str) -> None: ... + def record_run_end(self, run_id: str, status: str, stats: RunStats) -> None: ... + def record_item(self, item_id: str, run_id: str, slot_id: str, video_id: str, + family: str, task_type: str, skill_target: str, attempt: int, question_text: str) -> None: ... + def update_gates(self, item_id: str, report: GateReport) -> None: ... + def update_difficulty(self, item_id: str, difficulty_steps: int) -> None: ... + def get_run_stats(self, run_id: str) -> RunStats: ... + def close(self) -> None: ... +``` + +- [ ] **Step 1 (RED):** + +```python +class TestQuestionGenStore: + def test_schema_idempotent(self, store): ... + def test_record_run_lifecycle(self, store): ... # start -> end -> get_stats + def test_record_item_and_gates(self, store): ... + def test_update_difficulty(self, store): ... +``` + +- [ ] **Step 2 (GREEN):** 实现。同步 sqlite3（写入频率低无需 aiosqlite）。 + +- [ ] **Step 3 (COMMIT):** `feat(question_gen): add SQLite run store for generation telemetry` + +--- + +## Task 8: 重出循环 + 重量抽检 + v2 编排 `app/question_gen/pipeline_v2.py` + +**Files:** `app/question_gen/pipeline_v2.py`(新) | `tests/integration/test_pipeline_v2.py`(新) + +### 签名 + +```python +@dataclass(frozen=True) +class SlotAssignment: + slot_id: str; video_id: str; task_type: str; family: QuestionFamilySpec; seq: int + +@dataclass +class PipelineResult: + accepted: list[GeneratedQuestion]; rejected_count: int + heavy_sampled: list[tuple[str, int]] + +@dataclass(frozen=True) +class PipelineConfig: + family_ratios: dict[str, float]; per_type: int; retry_limit: int + heavy_sample_rate: float; dedup_threshold: float; concurrency: int + seed: int; output_dir: Path; gate_models: dict[str, str]; heavy_agent_model: str + +def load_pipeline_config(yaml_path: Path) -> PipelineConfig: ... + +def _assign_slots(video_ids: list[str], task_types: list[str], per_type: int, + family_ratios: dict[str, float], rng: random.Random) -> list[SlotAssignment]: ... + +async def _process_one_slot(slot, tree, vlm, llm, embed_fn, embed_pool, store, config, + used_node_ids, rng, sem, *, session_id) -> GeneratedQuestion | None: ... + # 生成 -> postprocess -> gates -> 重出(<=retry_limit) -> dedup -> accept/reject + +async def _heavy_check_one(question, tree, llm, *, session_id) -> int: ... + # 盲 Agent 试答 -> difficulty_steps + +async def run_pipeline_v2(video_ids, trees, vlm, llm, embed_fn, store, config, *, progress=None) -> PipelineResult: ... + # slots -> skip done -> sem-bounded gather -> heavy 15% -> stats +``` + +**_process_one_slot 核心流程:** + +``` +for attempt in 1..retry_limit: + material = sample_material_v2(...) # attempt>retry_limit 时重采样 + candidate = generate_one_v2(..., reject_reason=prev_reason) + store.record_item(...) + pp = run_postprocess(candidate.question, candidate.options, ...) + if pp.verbatim_ratio > 0.5: reject("verbatim") + report = await run_gates(candidate, tree, llm, family, pp) + store.update_gates(...) + if report.passed: + if not is_duplicate(candidate.question, embed_pool, embed_fn, threshold): + return to_generated_question(candidate) + prev_reason = report.reject_reason +return None +``` + +- [ ] **Step 1 (RED):** + +```python +class TestSlotAssignment: + def test_per_type_count(self): ... + def test_family_distribution(self): ... +class TestProcessOneSlot: + async def test_happy_path(self, deps): ... + async def test_retry_on_fail(self, deps): ... + async def test_max_retries_none(self, deps): ... +class TestPipelineV2: + async def test_full_flow(self, deps): ... + async def test_progress_resume(self, deps): ... + async def test_heavy_check_samples(self, deps): ... + async def test_store_records_all(self, deps): ... +``` + +- [ ] **Step 2 (GREEN):** 实现编排。 + +- [ ] **Step 3 (COMMIT):** `feat(question_gen): add v2 pipeline with retry loop and heavy check` + +--- + +## Task 9: CLI 改造 + sh 脚本 + 全量回归 + +**Files:** `tools/generate_questions.py`(改) | `scripts/generate_questions_v2.sh`(新) | `app/question_gen/__init__.py`(改) | `tests/integration/test_cli_generate_v2.py`(新) + +### 改造 + +```python +# tools/generate_questions.py 新增 +def _add_generate_v2_parser(subparsers) -> None: + p = subparsers.add_parser("generate-v2") + p.add_argument("--config", type=Path, default=Path("config/default.yaml")) + p.add_argument("--store-dir", type=Path, required=True) + p.add_argument("--db-path", type=Path, default=Path("logs/question_gen.db")) + p.add_argument("--seed", type=int, default=None) + p.add_argument("--dry-run", action="store_true") + +async def _run_generate_v2(args) -> None: ... + # load config -> build clients(DI) -> discover videos -> load trees -> init store -> resume progress -> run_pipeline_v2 -> cleanup +``` + +```bash +# scripts/generate_questions_v2.sh +#!/usr/bin/env bash +set -euo pipefail +STORE_DIR="${STORE_DIR:-store}"; CONFIG="${CONFIG:-config/default.yaml}" +DB_PATH="${DB_PATH:-logs/question_gen.db}" +source activate Video-Tree-TRM +[ "${MODE:-}" = "mock" ] && export LLM_MOCK=1 VLM_MOCK=1 +python tools/generate_questions.py generate-v2 --store-dir "$STORE_DIR" --config "$CONFIG" --db-path "$DB_PATH" ${SEED:+--seed $SEED} +``` + +- [ ] **Step 1 (RED):** + +```python +class TestCLIGenerateV2: + def test_subcommand_help(self): ... # returncode 0, "--config" in stdout + def test_dry_run(self, tmp_path): ... # 不调 LLM +``` + +- [ ] **Step 2 (GREEN):** 实现 CLI + sh。 + +- [ ] **Step 3:** 更新 `__init__.py` 导出: `run_pipeline_v2, PipelineConfig, PipelineResult, QuestionFamilySpec, ALL_FAMILIES, CandidateQuestion, generate_one_v2, GateReport, run_gates` + +- [ ] **Step 4:** 全量回归 + +```bash +conda activate Video-Tree-TRM && pytest tests/ --cov=app --cov=core --cov-report=term-missing -x +conda activate Video-Tree-TRM && ruff check app/ core/ adapters/ tools/ --fix && ruff format app/ core/ adapters/ tools/ +conda activate Video-Tree-TRM && radon cc app/question_gen/ -nc +``` + +- [ ] **Step 5 (COMMIT):** `feat(question_gen): add generate-v2 CLI subcommand and experiment script` + +--- + +## Self-Review + +### 设计合规性 + +| 设计决策 | 计划落点 | OK | +|----------|---------|:--:| +| 双标签 task_type + skill_target | T1 类型扩展 + T5 CandidateQuestion | Y | +| 5 题族 | T3 families.py | Y | +| 确定性后处理 | T2 postprocess.py（零 LLM） | Y | +| 轻量四门 ~4 LLM/题 | T6 gates.py | Y | +| 重出循环 max 3 + 拒因回填 | T8 _process_one_slot | Y | +| 重量抽检 15% | T8 _heavy_check_one | Y | +| SQLite 日志 | T7 run_store.py（schema 匹配 wiki） | Y | +| 输出 generated-v2/ | T8 config.output_dir | Y | +| 科研配置 YAML | T1 question_gen_v2 节 | Y | + +### 约束合规性 + +| 约束 | 验证 | +|------|------| +| 不改 harness/推理侧 | 全部新增在 app/question_gen/ | +| 保留 v1 generate_one | synthesizer.py 零改动 | +| GovernedLLMClient DI | 通过 LLMProvider/VLMProvider Protocol 注入 | +| 中文 docstring | 所有公共函数含中文文档 | +| TDD 红-绿-重构 | 每 Task 先 RED 再 GREEN | +| loguru 禁 print | 全部新模块 | +| 依赖方向 | 新模块在 app/ 层，仅依赖 core/ | + +### Codex Plan Review 修订记录 + +| 采纳 | 修订 | +|------|------| +| per_type 30→20 | 配置已修正为 20（12×20=240 硬约束） | +| pools.py 遗漏 | T1 补充注释：实现者须 grep _q_to_dict 并同步修改 | +| 出题禁区未覆盖 | T2 签名追加 `check_forbidden_material` 函数 | +| T6→T7 依赖缺失 | 拓扑图已添加边 | +| parent_call_id | 实现者在 T6/T9 中须贯穿 session_id → parent_call_id（GovernedLLMClient 已支持） | +| T8 过大 | 保留单 task 但实现者可按 _process_one_slot / _heavy_check_one / resume 分步 commit | +| heavy_check 复用推理管线 | 采纳为实施约束：T8 内 heavy_check 须调用现有 AgentLoop + 树环境，不重新实现 | + +驳回项： +- "采样约束覆盖不足"——这些是 prompt 模板内容（store/prompts/question_gen/），非代码逻辑；设计 §2.1 约束通过四门兜底验证 +- "T5 依赖 T2"——生成器输出原始答案位置，shuffle 在编排层（T8 pipeline_v2）后处理，不在生成器内 +- "chi-square 测试不稳定"——测试用固定 10000 次采样 + p>0.001 宽容阈值 + +### 核心算法保真校验 + +本计划**不涉及**算法清单 12 项中任何一项的修改。出题"gate"与训练 CE-Gate(#4) 完全不同；heavy_check 使用树搜索(#11)但不修改其实现。 + +### 风险与缓解 + +| 风险 | 缓解 | +|------|------| +| 四门 LLM 成本 | gate model 用 gpt-4.1-mini; asyncio.gather 并发 | +| 重出循环耗时 | retry_limit=3 硬上限 + 超限放弃 | +| heavy_check 超时 | GovernedLLMClient 超时 + 步数上限 | +| 族配比与 task_type 冲突 | legal_task_types 过滤后归一化 | diff --git a/research-wiki/schemas/question-gen-items.md b/research-wiki/schemas/question-gen-items.md new file mode 100644 index 0000000..a821157 --- /dev/null +++ b/research-wiki/schemas/question-gen-items.md @@ -0,0 +1,97 @@ +--- +type: schema +node_id: schema:question-gen-items +title: "表结构: question_gen_items（逐题门判定）" +date: 2026-07-12 +--- + +# 表结构: question_gen_items（逐题门判定） + +出题管线 v2 的逐题门判定记录。每题每轮（含重出）产生一行。 + +## 列定义 + +| 列名 | 类型 | 约束 | 说明 | +|------|------|------|------| +| `item_id` | TEXT | PK | 题目唯一 ID（UUID，每轮独立） | +| `run_id` | TEXT | NOT NULL FK→question_gen_runs | 关联批次 | +| `slot_id` | TEXT | NOT NULL | 逻辑 slot 标识（同 slot 多次重出共享） | +| `video_id` | TEXT | NOT NULL | 视频 ID | +| `family` | TEXT | NOT NULL | 题族（retrieval/reasoning/enumeration/visual/spatial） | +| `task_type` | TEXT | NOT NULL | Video-MME 12 类主标签 | +| `skill_target` | TEXT | | M1-M5 + 题族子标签 | +| `attempt` | INTEGER | NOT NULL | 当前重出轮次（1-based，≤3） | +| `gate_key_verify` | TEXT | | pass / reject / NULL（未执行） | +| `gate_blind_answer` | TEXT | | pass / reject / NULL | +| `gate_multi_true` | TEXT | | pass / reject / NULL | +| `gate_leak_test` | TEXT | | pass / reject / NULL | +| `reject_reason` | TEXT | | 拒因文本（首个拒绝门的判定摘要） | +| `final_status` | TEXT | NOT NULL | accepted / rejected / pending | +| `difficulty_steps` | INTEGER | | 重量抽检产出 Agent 步数（NULL=未抽检） | +| `created_at` | TEXT | NOT NULL | ISO8601 | + +## DDL + +```sql +CREATE TABLE IF NOT EXISTS question_gen_items ( + item_id TEXT PRIMARY KEY, + run_id TEXT NOT NULL REFERENCES question_gen_runs(run_id), + slot_id TEXT NOT NULL, + video_id TEXT NOT NULL, + family TEXT NOT NULL, + task_type TEXT NOT NULL, + skill_target TEXT, + attempt INTEGER NOT NULL DEFAULT 1, + gate_key_verify TEXT, + gate_blind_answer TEXT, + gate_multi_true TEXT, + gate_leak_test TEXT, + reject_reason TEXT, + final_status TEXT NOT NULL DEFAULT 'pending', + difficulty_steps INTEGER, + created_at TEXT NOT NULL +); + +CREATE INDEX idx_items_run ON question_gen_items(run_id); +CREATE INDEX idx_items_family ON question_gen_items(family); +CREATE INDEX idx_items_task_type ON question_gen_items(task_type); +CREATE INDEX idx_items_slot ON question_gen_items(slot_id); +``` + +## 埋点位置 + +| 代码位置 | 动作 | 时机 | +|---------|------|------| +| `synthesizer.py::generate_one_v2` 返回后 | INSERT（gates=NULL, status=pending） | 每题生成后 | +| `gates.py::run_gates` 返回后 | UPDATE gate_* 列 + reject_reason + final_status | 门跑完后 | +| 重出循环入口 | INSERT 新行（same slot_id, attempt+1） | 重出时 | +| `heavy_check.py` 完成 | UPDATE difficulty_steps | 抽检完成 | + +## 查询模式 + +```sql +-- 各门拦截率 +SELECT + gate_key_verify, + COUNT(*) as cnt +FROM question_gen_items WHERE run_id=? +GROUP BY gate_key_verify; + +-- 按题族切片拦截率 +SELECT family, + SUM(CASE WHEN final_status='rejected' THEN 1 ELSE 0 END) as rejected, + COUNT(*) as total +FROM question_gen_items WHERE run_id=? +GROUP BY family; + +-- 重出收敛率 +SELECT + COUNT(DISTINCT slot_id) FILTER (WHERE final_status='accepted') as converged, + COUNT(DISTINCT slot_id) as total_slots +FROM question_gen_items WHERE run_id=?; + +-- 按 task_type 分布 +SELECT task_type, COUNT(*) FROM question_gen_items +WHERE run_id=? AND final_status='accepted' +GROUP BY task_type; +``` diff --git a/research-wiki/schemas/question-gen-runs.md b/research-wiki/schemas/question-gen-runs.md new file mode 100644 index 0000000..79d729d --- /dev/null +++ b/research-wiki/schemas/question-gen-runs.md @@ -0,0 +1,58 @@ +--- +type: schema +node_id: schema:question-gen-runs +title: "表结构: question_gen_runs（出题批次）" +date: 2026-07-12 +--- + +# 表结构: question_gen_runs（出题批次） + +出题管线 v2 的批次级运行记录。每次 `tools/generate_questions.py generate` 执行产生一行。 + +## 列定义 + +| 列名 | 类型 | 约束 | 说明 | +|------|------|------|------| +| `run_id` | TEXT | PK | 生成批次唯一 ID（UUID） | +| `git_sha` | TEXT | NOT NULL | 代码版本（HEAD commit） | +| `config_snapshot` | TEXT | | 科研配置 YAML 快照（JSON 序列化） | +| `started_at` | TEXT | NOT NULL | ISO8601 开始时间 | +| `finished_at` | TEXT | | ISO8601 结束时间（运行中为 NULL） | +| `status` | TEXT | NOT NULL | running / completed / failed | +| `total_slots` | INTEGER | | 目标题数（默认 240） | +| `accepted` | INTEGER | | 过门题数 | +| `rejected` | INTEGER | | 最终拒绝题数 | +| `heavy_sampled` | INTEGER | | 重量抽检题数 | + +## DDL + +```sql +CREATE TABLE IF NOT EXISTS question_gen_runs ( + run_id TEXT PRIMARY KEY, + git_sha TEXT NOT NULL, + config_snapshot TEXT, + started_at TEXT NOT NULL, + finished_at TEXT, + status TEXT NOT NULL DEFAULT 'running', + total_slots INTEGER, + accepted INTEGER, + rejected INTEGER, + heavy_sampled INTEGER +); +``` + +## 埋点位置 + +| 代码位置 | 动作 | 时机 | +|---------|------|------| +| `tools/generate_questions.py::generate` 入口 | INSERT（status=running） | 批次开始 | +| `tools/generate_questions.py::generate` 结尾 | UPDATE（finished_at/status/统计汇总） | 批次结束 | + +## 查询模式 + +```sql +-- 最近一次成功运行 +SELECT * FROM question_gen_runs WHERE status='completed' ORDER BY started_at DESC LIMIT 1; +-- 产能统计 +SELECT accepted, rejected, heavy_sampled, total_slots FROM question_gen_runs WHERE run_id=?; +```