fix: address whole-impl review (INFRA T0 rows, reproducible manifest, evolution_target report, dead config, canonical DRY)

C-1: persist_infra_t0_rows 补 INFRA/空预测错题的 T0 信号行(不进诊断故须单独落库),run_pipeline 加 Phase 0,dry-run 用假数据走通。
C-2: CLI 加 --generated-at,真实运行默认盖真实 UTC now,可显式固定以字节级复现 manifest。
I-1: coverage_report 增 evolution_target_distribution(T2 信号按 tool/skill/system 计数)。
I-2: 删除 PoolConfig 死字段 n_trainval/floor_k/epsilon/report_floor/val_wrong_min(grep 确认无消费者,视频级切分用独立 VideoSplitConfig/SplitBuildConfig/SelectConfig)。
I-3: 抽共享 load_canonical_predictions(db_path, run_id),CLI 与 build_split 共用;消除 canonical 取行 + correct 判定重复。
M-1: build_split docstring 注明 val_wrong_min-agnostic 契约(McNemar 护栏由 CLI 冻结后执行,Task 11 契约)。
This commit is contained in:
2026-07-15 13:39:14 -04:00
parent 02b8145b7f
commit 8fef7ced42
6 changed files with 302 additions and 134 deletions
+64 -16
View File
@@ -119,6 +119,11 @@ def build_split(
test → 加载题库并以视频归属切三池 → 原子冻结 pools.json → 写溯源 manifest → test → 加载题库并以视频归属切三池 → 原子冻结 pools.json → 写溯源 manifest →
六条防御断言 fail-fast 校验。 六条防御断言 fail-fast 校验。
契约(Task 11,非疏漏):build_split 有意保持 val_wrong_min-agnostic——内部调
split_by_video_assignment 时不传 val_wrong_min(默认 0,不校验 validation 错题
数)。McNemar 功效护栏是切分**冻结后**的独立校验,由 CLI 的 check_mcnemar_power
在 build_split 返回后执行;切分构造本身不因功效阈失败,二者关注点分离。
参数: 参数:
db_path: harness.db 路径(只读读取 predictions,不改动)。 db_path: harness.db 路径(只读读取 predictions,不改动)。
baseline_run_id: 基线 run 标识(如 "infer_adhoc")。 baseline_run_id: 基线 run 标识(如 "infer_adhoc")。
@@ -138,7 +143,7 @@ def build_split(
ValueError: 上游依赖校验失败(如 correctness 缺题、assignment 非法)。 ValueError: 上游依赖校验失败(如 correctness 缺题、assignment 非法)。
""" """
# Phase 1: canonical 基线预测 + 诊断信号。 # Phase 1: canonical 基线预测 + 诊断信号。
preds = _read_canonical_predictions(db_path, baseline_run_id) preds = load_canonical_predictions(db_path, baseline_run_id)
signal_rows_raw = signal_store.load(baseline_run_id, diag_fingerprint) signal_rows_raw = signal_store.load(baseline_run_id, diag_fingerprint)
_assert_fingerprint_consistent(signal_rows_raw, diag_fingerprint) _assert_fingerprint_consistent(signal_rows_raw, diag_fingerprint)
signal_rows = [ signal_rows = [
@@ -226,31 +231,38 @@ def _normalize_choice(choice: str | None) -> str:
return (choice or "").strip().upper()[:1] return (choice or "").strip().upper()[:1]
def _read_canonical_predictions(db_path: Path, baseline_run_id: str) -> list[dict]: def load_canonical_predictions(db_path: Path, baseline_run_id: str) -> list[dict]:
"""从 harness.db 只读取指定 run 每题首行(ORDER BY rowid)为 canonical 预测。 """从 harness.db 只读取指定 run 每题首行(ORDER BY rowid)为 canonical 预测。
共享口径 helperCLI(可诊断错题筛选 + INFRA T0 补录)与 build_split(切分)
共用同一"每 qid 取 rowid 最小首行 + 归一化 correct 判定"口径,消除两处重复实现。
同一 question_id 可能有多行(重跑 / 补测),canonical 口径取 rowid 最小的首行, 同一 question_id 可能有多行(重跑 / 补测),canonical 口径取 rowid 最小的首行,
保证 distinct question 计数与对错判定确定。correct = 预测与答案归一后逐字符相等。 保证 distinct question 计数与对错判定确定。correct = 预测与答案各自归一
(strip → 大写 → 取首字母)后逐字符相等。
口径边界:旧 build_or_load_pools 的 legacy 池构建路径(app/harness/pools.py)是
另一条独立既有链路,不共用本 helper,两者刻意不统一(本次不动 legacy 路径)。
参数: 参数:
db_path: harness.db 路径。 db_path: harness.db 路径URI mode=ro 只读打开,绝不改动基线 db)
baseline_run_id: 基线 run 标识。 baseline_run_id: 基线 run 标识。
返回: 返回:
canonical 预测行列表,每行 {video_id, question_id, task_type, correct}。 canonical 预测行列表,每行含 question_id / video_id / task_type /
prediction / answer / stop_reason / correctbool)。按 rowid 升序去重,
每 qid 保留首行。
异常: 异常:
ValueError: 该 run 无任何预测行(fail-fast,不返回空切分)。 ValueError: 该 run 无任何预测行(fail-fast,不返回空切分)。
实现细节: 实现细节:
以 URI mode=ro 打开只读连接,绝不改动基线 db;按 rowid 升序遍历, 按 rowid 升序遍历,首次见到的 question_id 即 canonical 行,后续同 qid 行跳过。
首次见到的 question_id 即 canonical 行,后续同 qid 行跳过。
""" """
conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True) conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)
conn.row_factory = sqlite3.Row conn.row_factory = sqlite3.Row
try: try:
rows = conn.execute( rows = conn.execute(
"SELECT question_id, video_id, task_type, prediction, answer " "SELECT question_id, video_id, task_type, prediction, answer, stop_reason "
"FROM predictions WHERE run_id = ? ORDER BY rowid", "FROM predictions WHERE run_id = ? ORDER BY rowid",
(baseline_run_id,), (baseline_run_id,),
).fetchall() ).fetchall()
@@ -266,6 +278,9 @@ def _read_canonical_predictions(db_path: Path, baseline_run_id: str) -> list[dic
"question_id": qid, "question_id": qid,
"video_id": row["video_id"], "video_id": row["video_id"],
"task_type": row["task_type"], "task_type": row["task_type"],
"prediction": row["prediction"],
"answer": row["answer"],
"stop_reason": row["stop_reason"],
"correct": _normalize_choice(row["prediction"]) == _normalize_choice(row["answer"]), "correct": _normalize_choice(row["prediction"]) == _normalize_choice(row["answer"]),
} }
if not canonical: if not canonical:
@@ -367,7 +382,13 @@ def _build_coverage_report(
返回: 返回:
覆盖报告字典,含 cells_covered / grid_total / floor_satisfied / 覆盖报告字典,含 cells_covered / grid_total / floor_satisfied /
test_representativeness_deviation / tier_distribution test_representativeness_deviation / tier_distribution /
evolution_target_distribution。
实现细节:
evolution_target_distribution 只统计 T2 信号(可训练缺陷),按
tool / skill / system 计数,报告"哪层参数组拿到梯度"T0/T1/uncertain 行
evolution_target 恒为 None,不入该分布。
""" """
by_id = {v.video_id: v for v in videos} by_id = {v.video_id: v for v in videos}
trainval = [by_id[vid] for vid in assignment_obj.trainval] trainval = [by_id[vid] for vid in assignment_obj.trainval]
@@ -395,13 +416,7 @@ def _build_coverage_report(
diff_buckets, diff_buckets,
) )
tier_counts = Counter(row.tier for row in signal_rows_raw) tier_distribution, evolution_target_distribution = _signal_distributions(signal_rows_raw)
total_signals = sum(tier_counts.values())
tier_distribution = (
{tier: count / total_signals for tier, count in tier_counts.items()}
if total_signals
else {}
)
return { return {
"cells_covered": len(covered_cells), "cells_covered": len(covered_cells),
@@ -413,9 +428,42 @@ def _build_coverage_report(
"epsilon": config.epsilon, "epsilon": config.epsilon,
}, },
"tier_distribution": tier_distribution, "tier_distribution": tier_distribution,
"evolution_target_distribution": evolution_target_distribution,
} }
def _signal_distributions(
signal_rows_raw: list[DiagnosisSignalRow],
) -> tuple[dict[str, float], dict[str, int]]:
"""由诊断信号行算 tier 占比分布与 T2 进化目标计数分布。
参数:
signal_rows_raw: 诊断信号行。
返回:
(tier_distribution, evolution_target_distribution) 二元组:
- tier_distribution: {tier: 占比},无信号时为空 dict;
- evolution_target_distribution: 仅统计 T2(可训练缺陷)信号,按
tool / skill / system 计数,报告哪层参数组拿到梯度;T0/T1/uncertain 行
evolution_target 恒为 None,不入该分布。
"""
tier_counts = Counter(row.tier for row in signal_rows_raw)
total_signals = sum(tier_counts.values())
tier_distribution = (
{tier: count / total_signals for tier, count in tier_counts.items()}
if total_signals
else {}
)
evolution_target_distribution = dict(
Counter(
row.evolution_target
for row in signal_rows_raw
if row.tier == "T2" and row.evolution_target is not None
)
)
return tier_distribution, evolution_target_distribution
def _assert_split_invariants( def _assert_split_invariants(
*, *,
pools: Pools, pools: Pools,
+122 -48
View File
@@ -5,6 +5,14 @@
Phase 2 冻结切分(build_split,纯 code-controlled,产出 pools.json + manifest)→ Phase 2 冻结切分(build_split,纯 code-controlled,产出 pools.json + manifest)→
McNemar 功效护栏(validation 池错题数达阈校验)。 McNemar 功效护栏(validation 池错题数达阈校验)。
复现锚点约定(C-2):
- pools.json 的内容(+ seed + diag_fingerprint)是切分的**复现锚点**——相同输入
产出字节级相同的 pools.json 与 pools_sha256。
- manifest 的 generated_at 是**溯源元数据**,非复现锚点:真实运行默认盖真实 UTC
now(记录本次切分何时产出),但可用 `--generated-at <ISO>` 显式固定,以对
manifest 做字节级复现比对。write_manifest 库内不调 datetime.now,时间戳一律由
本 CLI 传入。
设计要点: 设计要点:
- 诊断口径指纹 = (诊断 prompt 版本, 模型名, git 短 SHA) 三分量合成,隔离不同 - 诊断口径指纹 = (诊断 prompt 版本, 模型名, git 短 SHA) 三分量合成,隔离不同
诊断配置的信号;换 prompt / 模型 / 代码实现即换指纹,旧信号不被覆盖。 诊断配置的信号;换 prompt / 模型 / 代码实现即换指纹,旧信号不被覆盖。
@@ -16,7 +24,8 @@
用于校验装配正确性(对齐 CLAUDE.md §2.5 smoke test)。 用于校验装配正确性(对齐 CLAUDE.md §2.5 smoke test)。
编排函数(run_pipeline)通过依赖注入接收 DiagnosisDeps / signal_store / wrong_ids / 编排函数(run_pipeline)通过依赖注入接收 DiagnosisDeps / signal_store / wrong_ids /
questions,便于单测用假实现替换、不触真实 LLM 与 harness.db。 questions / canonical_preds,便于单测用假实现替换、不触真实 LLM 与 harness.db。
其中 canonical_preds 供 Phase 0 补 INFRA / 空预测错题的 T0 信号(这些题不进诊断)。
""" """
from __future__ import annotations from __future__ import annotations
@@ -25,7 +34,6 @@ import argparse
import asyncio import asyncio
import datetime import datetime
import os import os
import sqlite3
import subprocess import subprocess
from dataclasses import dataclass from dataclasses import dataclass
from pathlib import Path from pathlib import Path
@@ -35,9 +43,15 @@ import yaml
from loguru import logger from loguru import logger
from app.harness.baseline_diagnosis import DiagnosisDeps, run_baseline_diagnosis from app.harness.baseline_diagnosis import DiagnosisDeps, run_baseline_diagnosis
from app.harness.build_split import SplitBuildConfig, SplitBuildResult, build_split from app.harness.build_split import (
SplitBuildConfig,
SplitBuildResult,
build_split,
load_canonical_predictions,
)
from app.harness.split_selection import diag_fingerprint from app.harness.split_selection import diag_fingerprint
from app.question_gen.loader import load_benchmark from app.question_gen.loader import load_benchmark
from core.evolution.types import DiagnosisSignalRow
if TYPE_CHECKING: if TYPE_CHECKING:
from app.harness.pools import Pools from app.harness.pools import Pools
@@ -351,57 +365,81 @@ def _load_diagnose_prompts() -> Any:
) )
def _normalize_choice(choice: str | None) -> str: def select_diagnosable_wrong_ids(preds: list[dict]) -> list[str]:
"""选项归一:strip → 大写 → 取首字母(None 归一为空串)。""" """从 canonical 预测筛出可诊断错题 question_id(保序)。
return (choice or "").strip().upper()[:1]
可诊断错题判据:预测非空 且 stop_reason 非 INFRAerror / parse_error)且
def load_diagnosable_wrong_ids(harness_db: Path, baseline_run_id: str) -> list[str]: 归一后预测 != 答案。INFRA / 空预测错题不进 wrong_ids——它们改由
"""从 harness.db 读 baseline run 的可诊断错题 question_id(保序、canonical 首行)。 persist_infra_t0_rows 直接落 T0run_diagnosis 内部也会二次排除同类题)。
可诊断错题判据:canonical 首行(rowid 最小)预测非空 且 stop_reason 非 INFRA
error / parse_error)且 归一后预测 != 答案。INFRA / 空预测题不进 wrong_ids
run_diagnosis 内部也会二次排除,此处前置过滤减少无谓 LLM 调用)。
参数: 参数:
harness_db: harness.db 路径(只读打开)。 preds: load_canonical_predictions 产出的 canonical 预测行(已按 qid 去重)。
baseline_run_id: 基线 run 标识。
返回: 返回:
可诊断错题 question_id 列表(按 rowid 升序 canonical 顺序,去重)。 可诊断错题 question_id 列表(保 preds 顺序)。
异常:
SystemExit: 该 run 无任何预测行(fail loud)。
""" """
conn = sqlite3.connect(f"file:{harness_db}?mode=ro", uri=True)
conn.row_factory = sqlite3.Row
try:
rows = conn.execute(
"SELECT question_id, prediction, answer, stop_reason "
"FROM predictions WHERE run_id = ? ORDER BY rowid",
(baseline_run_id,),
).fetchall()
finally:
conn.close()
if not rows:
raise SystemExit(
f"run_id={baseline_run_id}{harness_db} 无任何预测行,无法诊断(P5 fail loud"
)
seen: set[str] = set()
wrong_ids: list[str] = [] wrong_ids: list[str] = []
for row in rows: for pred in preds:
qid = row["question_id"] prediction = (pred["prediction"] or "").strip()
if qid in seen: if not prediction or pred["stop_reason"] in _INFRA_STOP_REASONS:
continue continue
seen.add(qid) if not pred["correct"]:
prediction = (row["prediction"] or "").strip() wrong_ids.append(pred["question_id"])
if not prediction or row["stop_reason"] in _INFRA_STOP_REASONS:
continue
if _normalize_choice(row["prediction"]) != _normalize_choice(row["answer"]):
wrong_ids.append(qid)
return wrong_ids return wrong_ids
def persist_infra_t0_rows(
store: DiagnosisSignalStore,
preds: list[dict],
baseline_run_id: str,
diag_fingerprint: str,
) -> int:
"""把非正确且 INFRA / 空预测的错题以 T0 信号行 upsert 落库(幂等)。
这些题(stop_reason ∈ {error, parse_error} 或预测为空)从不进入 run_diagnosis
(筛选时被前置排除),故其 T0 信号必须在此单独补齐——否则 signal store 缺这些行,
tier 分布 / manifest 不完整(计划要求 4 个 INFRA 空预测错题 → T0)。
投影口径与 baseline_diagnosis 的 INFRA 投影一致:infra=True、tier="T0"
error_type / cause_category / evolution_target 均 None、degraded=False
video_id / task_type 从 canonical 预测取。store.upsert 按主键
(question_id, baseline_run_id, diag_fingerprint) 幂等,重复调用零副作用。
参数:
store: 诊断信号存储端口(与诊断落库同一 store)。
preds: load_canonical_predictions 产出的 canonical 预测行。
baseline_run_id: 基线 run 标识(信号行主键之一)。
diag_fingerprint: 诊断口径指纹(信号行主键之一)。
返回:
落库的 T0 行数(供日志)。
"""
count = 0
for pred in preds:
prediction = (pred["prediction"] or "").strip()
is_infra_or_empty = pred["stop_reason"] in _INFRA_STOP_REASONS or not prediction
if pred["correct"] or not is_infra_or_empty:
continue
store.upsert(
DiagnosisSignalRow(
question_id=pred["question_id"],
video_id=pred["video_id"],
baseline_run_id=baseline_run_id,
diag_fingerprint=diag_fingerprint,
task_type=pred["task_type"],
error_type=None,
cause_category=None,
tier="T0",
evolution_target=None,
degraded=False,
infra=True,
session_id=None,
)
)
count += 1
return count
def load_questions_by_id(questions_dir: Path) -> dict[str, GeneratedQuestion]: def load_questions_by_id(questions_dir: Path) -> dict[str, GeneratedQuestion]:
"""加载 benchmark 全部题并建 question_id → GeneratedQuestion 映射。 """加载 benchmark 全部题并建 question_id → GeneratedQuestion 映射。
@@ -460,28 +498,34 @@ async def run_pipeline(
signal_store: DiagnosisSignalStore, signal_store: DiagnosisSignalStore,
wrong_ids: list[str], wrong_ids: list[str],
questions: dict[str, GeneratedQuestion], questions: dict[str, GeneratedQuestion],
canonical_preds: list[dict],
harness_db: Path, harness_db: Path,
questions_dir: Path, questions_dir: Path,
out_dir: Path, out_dir: Path,
generated_at: str, generated_at: str,
) -> SplitBuildResult: ) -> SplitBuildResult:
"""内联阶段:Phase 1 诊断 → Phase 2 冻结切分 → McNemar 护栏。 """内联阶段:Phase 0 INFRA T0 补录 → Phase 1 诊断 → Phase 2 冻结切分 → McNemar 护栏。
参数: 参数:
config: 科研旋钮快照。 config: 科研旋钮快照。
fingerprint: 诊断口径指纹(已合成,作诊断信号主键之一)。 fingerprint: 诊断口径指纹(已合成,作诊断信号主键之一)。
diagnosis_deps: Phase 1 诊断依赖束(真实或假实现)。 diagnosis_deps: Phase 1 诊断依赖束(真实或假实现)。
signal_store: 诊断信号存储端口(Phase 1 写、Phase 2 读)。 signal_store: 诊断信号存储端口(Phase 0/1 写、Phase 2 读)。
wrong_ids: 待诊断的可诊断错题 question_id 列表。 wrong_ids: 待诊断的可诊断错题 question_id 列表。
questions: question_id → GeneratedQuestion 映射。 questions: question_id → GeneratedQuestion 映射。
canonical_preds: canonical 预测行(Phase 0 从中筛 INFRA / 空预测错题补 T0)。
harness_db: harness.db 路径(Phase 2 读 canonical 预测)。 harness_db: harness.db 路径(Phase 2 读 canonical 预测)。
questions_dir: benchmark 题库目录(Phase 2 加载题库切池)。 questions_dir: benchmark 题库目录(Phase 2 加载题库切池)。
out_dir: 冻结产物目录(pools.json + split_manifest.json)。 out_dir: 冻结产物目录(pools.json + split_manifest.json)。
generated_at: 生成时间戳(ISO 字符串,由调用方传入保证可复现)。 generated_at: 生成时间戳(ISO 字符串,由调用方传入;见模块 C-2 复现锚点约定)。
返回: 返回:
SplitBuildResult(冻结三池 + manifest + assignment)。 SplitBuildResult(冻结三池 + manifest + assignment)。
""" """
# Phase 0: INFRA / 空预测错题补 T0(这些题不进诊断,须单独落库保证 tier 分布/manifest 完整)。
n_t0 = persist_infra_t0_rows(signal_store, canonical_preds, config.baseline_run_id, fingerprint)
logger.info("Phase 0 INFRA T0 补录:落库 {} 行(INFRA / 空预测错题不进诊断)", n_t0)
# Phase 1: 离线诊断(断点续跑幂等:done_question_ids 已完成题跳过)。 # Phase 1: 离线诊断(断点续跑幂等:done_question_ids 已完成题跳过)。
logger.info( logger.info(
"Phase 1 离线诊断:baseline={} 待诊断错题 {}", config.baseline_run_id, len(wrong_ids) "Phase 1 离线诊断:baseline={} 待诊断错题 {}", config.baseline_run_id, len(wrong_ids)
@@ -548,12 +592,16 @@ def _execute_real(config: VideoSplitConfig, fingerprint: str, args: argparse.Nam
harness_db, questions_dir, out_dir = _resolve_paths(args) harness_db, questions_dir, out_dir = _resolve_paths(args)
if not harness_db.exists(): if not harness_db.exists():
raise SystemExit(f"harness.db 不存在: {harness_db}P5 fail loud") raise SystemExit(f"harness.db 不存在: {harness_db}P5 fail loud")
wrong_ids = load_diagnosable_wrong_ids(harness_db, config.baseline_run_id) canonical_preds = load_canonical_predictions(harness_db, config.baseline_run_id)
wrong_ids = select_diagnosable_wrong_ids(canonical_preds)
questions = load_questions_by_id(questions_dir) questions = load_questions_by_id(questions_dir)
deps = build_diagnosis_deps( deps = build_diagnosis_deps(
harness_db=harness_db, concurrency=args.concurrency, expected_model=config.model harness_db=harness_db, concurrency=args.concurrency, expected_model=config.model
) )
# generated_at:默认盖真实 UTC now(溯源用),--generated-at 可显式固定以复现(C-2)。
generated_at = args.generated_at or datetime.datetime.now(datetime.UTC).isoformat()
from adapters.baseline_diagnosis_store import SqliteDiagnosisSignalStore from adapters.baseline_diagnosis_store import SqliteDiagnosisSignalStore
store = SqliteDiagnosisSignalStore(str(harness_db)) store = SqliteDiagnosisSignalStore(str(harness_db))
@@ -566,10 +614,11 @@ def _execute_real(config: VideoSplitConfig, fingerprint: str, args: argparse.Nam
signal_store=store, signal_store=store,
wrong_ids=wrong_ids, wrong_ids=wrong_ids,
questions=questions, questions=questions,
canonical_preds=canonical_preds,
harness_db=harness_db, harness_db=harness_db,
questions_dir=questions_dir, questions_dir=questions_dir,
out_dir=out_dir, out_dir=out_dir,
generated_at=datetime.datetime.now(datetime.UTC).isoformat(), generated_at=generated_at,
) )
) )
finally: finally:
@@ -637,6 +686,20 @@ def _execute_dry_run(config: VideoSplitConfig, fingerprint: str, args: argparse.
dry_db.parent.mkdir(parents=True, exist_ok=True) dry_db.parent.mkdir(parents=True, exist_ok=True)
store = SqliteDiagnosisSignalStore(str(dry_db)) store = SqliteDiagnosisSignalStore(str(dry_db))
try: try:
# Phase 0 装配:用一条假 INFRA 空预测走通 persist_infra_t0_rows(不触 LLM)。
fake_infra_preds = [
{
"question_id": "_dry_infra",
"video_id": "_dry_v",
"task_type": "Counting Problem",
"prediction": "",
"answer": "A",
"stop_reason": "error",
"correct": False,
}
]
n_t0 = persist_infra_t0_rows(store, fake_infra_preds, config.baseline_run_id, fingerprint)
logger.info("Phase 0 装配 OKpersist_infra_t0_rows 落 {} 行 INFRA T0(假数据)", n_t0)
logger.info("Phase 1 装配 OKrun_baseline_diagnosis 以空错题走早返回路径(不触 LLM)") logger.info("Phase 1 装配 OKrun_baseline_diagnosis 以空错题走早返回路径(不触 LLM)")
asyncio.run( asyncio.run(
run_baseline_diagnosis( run_baseline_diagnosis(
@@ -667,6 +730,17 @@ def build_arg_parser() -> argparse.ArgumentParser:
parser.add_argument("--harness-db", type=Path, default=None, dest="harness_db") parser.add_argument("--harness-db", type=Path, default=None, dest="harness_db")
parser.add_argument("--questions-dir", type=Path, default=None, dest="questions_dir") parser.add_argument("--questions-dir", type=Path, default=None, dest="questions_dir")
parser.add_argument("--out-dir", type=Path, default=None, dest="out_dir") parser.add_argument("--out-dir", type=Path, default=None, dest="out_dir")
parser.add_argument(
"--generated-at",
type=str,
default=None,
dest="generated_at",
help=(
"manifest generated_at 时间戳(ISO 字符串);默认盖真实 UTC now(溯源元数据)。"
"复现锚点是 pools.json 内容 + seed + fingerprintgenerated_at 可显式传入以"
"对 manifest 做字节级复现比对。"
),
)
return parser return parser
+4 -13
View File
@@ -160,16 +160,12 @@ class PoolConfig:
eval_min_per_class: 验证池中每类保底样本数(GlobalStrategy 用)。 eval_min_per_class: 验证池中每类保底样本数(GlobalStrategy 用)。
train_ratio: train/(train+val) 比例(PerCategoryStrategy 用)。 train_ratio: train/(train+val) 比例(PerCategoryStrategy 用)。
test_questions_dir: 外部 test 题源路径(PerCategoryStrategy 用)。 test_questions_dir: 外部 test 题源路径(PerCategoryStrategy 用)。
n_trainval: trainval 目标视频数(结果驱动视频级切分用;0 表示不启用)。
floor_k: 各高信号 task_type 的 T2 defect 下限(视频级切分硬约束;空表示无约束)。
epsilon: test 相对全局的最大允许分布偏差(视频级切分 test 代表性守护)。
report_floor: per-type 报告门限,题数 ≥ 此值的 task_type 才入 ε 约束(视频级切分用)。
val_wrong_min: validation 池最少错题数(McNemar 功效护栏;0 表示不检查)。
实现细节: 实现细节:
视频级切分五个旋钮均带惰性默认(0 / 空 dict),使现有 GlobalPoolStrategy / 结果驱动视频级切分不复用本配置——它有独立的 VideoSplitConfig /
PerCategoryStrategy 的构造点无需改动即可保持行为不变。floor_k 为不可哈希 dict SplitBuildConfig / SelectConfig(见 app/harness/video_split_cli.py 与
标 hash=False 排除出 frozen dataclass 的自动 __hash__,避免入 set/dict 键时报错。 split_selection.py),故本类不承载 n_trainval / floor_k / epsilon 等视频级
切分旋钮,避免死配置面。
""" """
task_types: tuple[str, ...] | None task_types: tuple[str, ...] | None
@@ -184,8 +180,3 @@ class PoolConfig:
train_ratio: float train_ratio: float
test_questions_dir: _Path | None test_questions_dir: _Path | None
batch_correct_ratio: float | None = None batch_correct_ratio: float | None = None
n_trainval: int = 0
floor_k: dict[str, int] = field(default_factory=dict, hash=False)
epsilon: float = 0.0
report_floor: int = 0
val_wrong_min: int = 0
@@ -153,3 +153,10 @@ def test_end_to_end_freezes_valid_pools(tmp_path: Path) -> None:
assert coverage["grid_total"] == 48 assert coverage["grid_total"] == 48
assert "tier_distribution" in coverage assert "tier_distribution" in coverage
assert manifest_path.exists() assert manifest_path.exists()
# I-1evolution_target_distribution 出现在 coverage_report,且 T2 信号按 tool/skill/system 计数。
target_dist = coverage["evolution_target_distribution"]
assert set(target_dist) <= {"tool", "skill", "system"}
# 全部错题构造为 T2(error_type 四类轮转),进化目标覆盖三层且计数为正。
assert sum(target_dist.values()) > 0
assert set(target_dist) == {"tool", "skill", "system"}
+6 -53
View File
@@ -1,10 +1,12 @@
"""视频级切分科研旋钮单元测试:诊断指纹 + val_wrong_min 功效护栏 + PoolConfig 新字段 """视频级切分科研旋钮单元测试:诊断指纹 + val_wrong_min 功效护栏。
覆盖: 覆盖:
- diag_fingerprint 对 (prompt 版本 / 模型 / 代码版本) 三元组确定且敏感; - diag_fingerprint 对 (prompt 版本 / 模型 / 代码版本) 三元组确定且敏感;
- split_by_video_assignment 的 val_wrong_min 门控 fail loud(验证信号不足即报错); - split_by_video_assignment 的 val_wrong_min 门控 fail loud(验证信号不足即报错);
- val_wrong_min 默认 0 时行为与 Task 11 现有调用完全一致(不回归) - val_wrong_min 默认 0 时行为与 Task 11 现有调用完全一致(不回归)
- PoolConfig 能接收视频级切分的五个新旋钮字段(纯 dataclass 装配)。
注:结果驱动视频级切分不复用 PoolConfig——它有独立的 VideoSplitConfig /
SplitBuildConfig / SelectConfig,故 PoolConfig 不承载视频级切分旋钮(无死配置面)。
""" """
from __future__ import annotations from __future__ import annotations
@@ -13,7 +15,7 @@ import pytest
from app.harness.pools import InsufficientValSignal, split_by_video_assignment from app.harness.pools import InsufficientValSignal, split_by_video_assignment
from app.harness.split_selection import diag_fingerprint from app.harness.split_selection import diag_fingerprint
from core.types import GeneratedQuestion, PoolConfig from core.types import GeneratedQuestion
def _q(qid: str, vid: str, tt: str = "Counting Problem") -> GeneratedQuestion: def _q(qid: str, vid: str, tt: str = "Counting Problem") -> GeneratedQuestion:
@@ -64,52 +66,3 @@ def test_val_wrong_min_default_zero_no_regression():
qs, assignment, correctness=correctness, val_ratio=1.0, seed=0 qs, assignment, correctness=correctness, val_ratio=1.0, seed=0
) )
assert len(pools.validation) == 2 # 未抛异常,正常返回 assert len(pools.validation) == 2 # 未抛异常,正常返回
def test_pool_config_accepts_video_split_knobs():
"""PoolConfig 能接收视频级切分五个新旋钮字段(默认惰性,不破坏现有构造点)。"""
cfg = PoolConfig(
task_types=None,
seed=0,
baseline_run_id="infer_adhoc",
diag_size=200,
diag_correct_ratio=0.5,
val_size=30,
val_correct_ratio=0.5,
test_size=60,
eval_min_per_class=2,
train_ratio=0.667,
test_questions_dir=None,
n_trainval=100,
floor_k={"Counting Problem": 3},
epsilon=0.1,
report_floor=27,
val_wrong_min=20,
)
assert cfg.n_trainval == 100
assert cfg.floor_k == {"Counting Problem": 3}
assert cfg.epsilon == 0.1
assert cfg.report_floor == 27
assert cfg.val_wrong_min == 20
def test_pool_config_video_split_knobs_default_inert():
"""未传视频级切分字段时默认惰性(0 / 空 dict),不破坏 GlobalPoolStrategy 现有构造。"""
cfg = PoolConfig(
task_types=None,
seed=0,
baseline_run_id="run_1",
diag_size=200,
diag_correct_ratio=0.5,
val_size=30,
val_correct_ratio=0.5,
test_size=60,
eval_min_per_class=2,
train_ratio=0.667,
test_questions_dir=None,
)
assert cfg.n_trainval == 0
assert cfg.floor_k == {}
assert cfg.epsilon == 0.0
assert cfg.report_floor == 0
assert cfg.val_wrong_min == 0
+99 -4
View File
@@ -102,10 +102,18 @@ def test_check_mcnemar_power_zero_threshold_skips():
assert cli.check_mcnemar_power(pools, val_wrong_min=0) == 0 assert cli.check_mcnemar_power(pools, val_wrong_min=0) == 0
def test_run_pipeline_orders_two_phases(monkeypatch, tmp_path): def test_run_pipeline_orders_three_phases(monkeypatch, tmp_path):
"""run_pipeline 先 Phase 1 诊断、后 Phase 2 build_split(按序)。""" """run_pipeline 先补 Phase 0 INFRA T0、再 Phase 1 诊断、后 Phase 2 build_split(按序)。"""
calls: list[str] = [] calls: list[str] = []
class _SpyStore:
def __init__(self):
self.t0_rows: list = []
def upsert(self, row):
calls.append("t0_upsert")
self.t0_rows.append(row)
async def fake_diag(**kwargs): async def fake_diag(**kwargs):
calls.append("diagnosis") calls.append("diagnosis")
assert kwargs["diag_fingerprint"] == "fp" assert kwargs["diag_fingerprint"] == "fp"
@@ -120,24 +128,111 @@ def test_run_pipeline_orders_two_phases(monkeypatch, tmp_path):
monkeypatch.setattr(cli, "run_baseline_diagnosis", fake_diag) monkeypatch.setattr(cli, "run_baseline_diagnosis", fake_diag)
monkeypatch.setattr(cli, "build_split", fake_build_split) monkeypatch.setattr(cli, "build_split", fake_build_split)
# 一条 INFRA 空预测错题 → Phase 0 应补一行 T0(在诊断/切分之前)。
canonical_preds = [
{
"question_id": "q_infra",
"video_id": "v9",
"task_type": "Counting Problem",
"prediction": "",
"answer": "A",
"stop_reason": "error",
"correct": False,
}
]
store = _SpyStore()
result = asyncio.run( result = asyncio.run(
cli.run_pipeline( cli.run_pipeline(
config=_config(), config=_config(),
fingerprint="fp", fingerprint="fp",
diagnosis_deps=object(), diagnosis_deps=object(),
signal_store=object(), signal_store=store,
wrong_ids=["q1"], wrong_ids=["q1"],
questions={}, questions={},
canonical_preds=canonical_preds,
harness_db=tmp_path / "h.db", harness_db=tmp_path / "h.db",
questions_dir=tmp_path, questions_dir=tmp_path,
out_dir=tmp_path / "out", out_dir=tmp_path / "out",
generated_at="2026-07-15T00:00:00Z", generated_at="2026-07-15T00:00:00Z",
) )
) )
assert calls == ["diagnosis", "build_split"] assert calls == ["t0_upsert", "diagnosis", "build_split"] # Phase 0 先于诊断与切分
assert len(store.t0_rows) == 1
assert store.t0_rows[0].tier == "T0" and store.t0_rows[0].infra is True
assert result.pools.validation == [] assert result.pools.validation == []
def test_select_diagnosable_wrong_ids_excludes_infra_and_correct():
"""可诊断错题筛选:排除 INFRA / 空预测 / 正确题,保留非空非 INFRA 错题(保序)。"""
preds = [
{"question_id": "ok", "stop_reason": "finished", "prediction": "B", "correct": True},
{"question_id": "wrong", "stop_reason": "finished", "prediction": "C", "correct": False},
{"question_id": "infra", "stop_reason": "error", "prediction": "", "correct": False},
{"question_id": "parse", "stop_reason": "parse_error", "prediction": "x", "correct": False},
{"question_id": "empty", "stop_reason": "finished", "prediction": "", "correct": False},
]
assert cli.select_diagnosable_wrong_ids(preds) == ["wrong"]
def test_persist_infra_t0_rows_persists_only_infra_or_empty_wrong(tmp_path):
"""INFRA / 空预测错题落 T0infra=Trueerror_type/target=None);正确 / 可诊断错题不落。"""
from adapters.baseline_diagnosis_store import SqliteDiagnosisSignalStore
preds = [
{
"question_id": "ok",
"video_id": "v1",
"task_type": "Counting Problem",
"prediction": "A",
"answer": "A",
"stop_reason": "finished",
"correct": True,
},
{
"question_id": "diag_wrong",
"video_id": "v2",
"task_type": "Counting Problem",
"prediction": "C",
"answer": "A",
"stop_reason": "finished",
"correct": False,
},
{
"question_id": "infra_err",
"video_id": "v3",
"task_type": "OCR Problems",
"prediction": "",
"answer": "A",
"stop_reason": "error",
"correct": False,
},
{
"question_id": "parse_err",
"video_id": "v4",
"task_type": "Counting Problem",
"prediction": "",
"answer": "A",
"stop_reason": "parse_error",
"correct": False,
},
]
store = SqliteDiagnosisSignalStore(str(tmp_path / "h.db"))
n = cli.persist_infra_t0_rows(store, preds, "infer_adhoc", "fp")
assert n == 2 # 只有两条 INFRA 空预测错题
rows = {r.question_id: r for r in store.load("infer_adhoc", "fp")}
assert set(rows) == {"infra_err", "parse_err"}
for r in rows.values():
assert r.tier == "T0" and r.infra is True
assert r.error_type is None and r.evolution_target is None and r.cause_category is None
assert r.degraded is False
# 幂等:重复调用同 PK 覆盖,行数不变。
assert cli.persist_infra_t0_rows(store, preds, "infer_adhoc", "fp") == 2
assert len(store.load("infer_adhoc", "fp")) == 2
store.close()
def test_dry_run_computes_fingerprint_without_llm(monkeypatch, tmp_path, capsys): def test_dry_run_computes_fingerprint_without_llm(monkeypatch, tmp_path, capsys):
"""--dry-rundiag_fingerprint 被调用、Phase 1 走空错题早返回、不真调 LLM。""" """--dry-rundiag_fingerprint 被调用、Phase 1 走空错题早返回、不真调 LLM。"""
fp_calls: list[tuple[str, str, str]] = [] fp_calls: list[tuple[str, str, str]] = []