267 lines
11 KiB
Python
267 lines
11 KiB
Python
"""离线诊断编排：把 baseline run 的错题诊断投影为逐题信号行并断点续跑落库。
|
||
|
||
"结果驱动视频级切分"离线管线的诊断步。给定一批可诊断错题：
|
||
1. 算 remaining（跳过 store 已完成题）实现续跑幂等；
|
||
2. 对剩余错题调 core.evolution.diagnose.run_diagnosis（经 StepsJsonRunLog
|
||
包装内层 RunLog，兼容 traces 未落表的历史 run）；
|
||
3. 把 error_attributions / infra / degraded 三类产物确定性投影为
|
||
DiagnosisSignalRow（tier 由 split_selection.score_signal 判定）；
|
||
4. run 末（Phase 3）逐行 store.upsert 落库——诊断在 Phase 2 全部跑完后才落库，
|
||
故崩溃丢本次 run 未落库的全部结果（不是"仅一行"）；靠 GovernedLLMClient 的
|
||
Redis 缓存缓解重跑时的 LLM 重烧，下次调用命中缓存直接续。
|
||
|
||
错误处理诚实标注（不谎称全传播）:
|
||
- run_diagnosis 的 C1/C2 阶段（指标计算、错误归因）网络/API 失败经
|
||
GovernedLLMClient 重试栈后仍失败会向上抛出，本编排不捕获、不掩盖，
|
||
直接冒泡给调用方。
|
||
- 但 C3 阶段（defect/lapse judge）的调用整体包在 `except Exception` 内
|
||
（core/evolution/diagnose.py:2186），故 C3 judge 的**全部异常（含网络/API
|
||
失败）都被吞并→warning→默认归为 lapse**，不会向上抛；judge 语义歧义同样
|
||
按此保护性 fallback 处理。本编排原样接受该判定，不二次兜底、也不谎称
|
||
C3 阶段网络失败会传播。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from dataclasses import dataclass
|
||
from typing import TYPE_CHECKING, Any
|
||
|
||
from loguru import logger
|
||
|
||
from app.harness.baseline_run_log import StepsJsonRunLog
|
||
from app.harness.split_selection import evolution_target_of, score_signal
|
||
from core.evolution.diagnose import run_diagnosis
|
||
from core.evolution.types import DiagnosisSignalRow
|
||
|
||
if TYPE_CHECKING:
|
||
from core.evolution.protocols import DiagnosisSignalStore
|
||
from core.evolution.types import DiagnosisResult
|
||
from core.types import GeneratedQuestion
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class DiagnosisDeps:
|
||
"""离线诊断编排的依赖束（一次编排的全部外部端口 + 运行参数）。
|
||
|
||
frozen 保证一次编排内依赖不可变；LLM/RunLog/SkillStore/prompts 走 Protocol
|
||
注入，便于测试替换成假实现。
|
||
|
||
属性:
|
||
run_log: 内层 RunLog 实现（提供 get_predictions/get_traces），
|
||
编排内部再用 StepsJsonRunLog 包装以兼容 traces 未落表的 run。
|
||
llm: LLM 调用端口（治理后的 GovernedLLMClient）。
|
||
skill_store: 技能文件读取端口。
|
||
prompts: 诊断模板束（DiagnosePrompts）。
|
||
tree_data: 树结构字典（多视频 {video_id: tree} 或单棵树），透传给 run_diagnosis。
|
||
concurrency: 诊断并发上限。
|
||
"""
|
||
|
||
run_log: Any
|
||
llm: Any
|
||
skill_store: Any
|
||
prompts: Any
|
||
tree_data: dict[str, Any]
|
||
concurrency: int
|
||
|
||
|
||
async def run_baseline_diagnosis(
|
||
*,
|
||
baseline_run_id: str,
|
||
diag_fingerprint: str,
|
||
wrong_ids: list[str],
|
||
questions: dict[str, GeneratedQuestion],
|
||
store: DiagnosisSignalStore,
|
||
deps: DiagnosisDeps,
|
||
retry_uncertain: bool = False,
|
||
) -> None:
|
||
"""对 baseline run 的错题跑离线诊断并把信号落库（断点续跑幂等）。
|
||
|
||
参数:
|
||
baseline_run_id: baseline run 标识（如 "infer_adhoc"），信号行主键之一。
|
||
diag_fingerprint: 诊断口径指纹，隔离不同诊断配置的信号，主键之一。
|
||
wrong_ids: 本次待诊断的可诊断错题 question_id 列表（保序）。
|
||
questions: question_id → GeneratedQuestion 映射，需覆盖 wrong_ids 全部题
|
||
及 run_diagnosis 返回的所有 infra/degraded 题（用于取 video_id/task_type）。
|
||
store: 诊断信号存储端口，upsert 落盘并提供 done_question_ids 续跑查询。
|
||
deps: 外部依赖束（见 DiagnosisDeps）。
|
||
retry_uncertain: True 时把已落 tier='uncertain'（信号不可信降级）的题也纳入
|
||
remaining 重新诊断，透传给 store.done_question_ids；默认 False。
|
||
|
||
返回:
|
||
None。副作用为把逐题 DiagnosisSignalRow 写入 store。
|
||
|
||
关键实现:
|
||
- remaining = wrong_ids 去除 store 已完成题；空则直接 return（续跑幂等，
|
||
重复调用零副作用）。
|
||
- run_diagnosis 只诊断 remaining，避免重复 LLM 调用浪费。
|
||
- 三类产物投影互斥落库：error_attributions（defect/lapse）、infra_question_ids
|
||
（T0）、degraded_question_ids（uncertain）。
|
||
"""
|
||
# Phase 1: 算 remaining（续跑幂等）
|
||
done = store.done_question_ids(
|
||
baseline_run_id, diag_fingerprint, retry_uncertain=retry_uncertain
|
||
)
|
||
remaining = [qid for qid in wrong_ids if qid not in done]
|
||
if not remaining:
|
||
logger.info(
|
||
"离线诊断续跑：baseline={} fingerprint={} 无剩余错题（已完成 {} 题），跳过。",
|
||
baseline_run_id,
|
||
diag_fingerprint,
|
||
len(done),
|
||
)
|
||
return
|
||
|
||
logger.info(
|
||
"离线诊断开始：baseline={} fingerprint={} 剩余 {}/{} 题待诊断。",
|
||
baseline_run_id,
|
||
diag_fingerprint,
|
||
len(remaining),
|
||
len(wrong_ids),
|
||
)
|
||
|
||
# Phase 2: 对剩余错题跑诊断（StepsJsonRunLog 兼容 traces 未落表的历史 run）
|
||
result = await run_diagnosis(
|
||
baseline_run_id,
|
||
[questions[qid] for qid in remaining],
|
||
deps.tree_data,
|
||
deps.llm,
|
||
StepsJsonRunLog(deps.run_log),
|
||
deps.skill_store,
|
||
deps.prompts,
|
||
concurrency=deps.concurrency,
|
||
question_ids=list(remaining),
|
||
only_incorrect=True,
|
||
)
|
||
|
||
# Phase 3: 投影落库
|
||
counts = _project_and_persist(
|
||
result=result,
|
||
baseline_run_id=baseline_run_id,
|
||
diag_fingerprint=diag_fingerprint,
|
||
questions=questions,
|
||
store=store,
|
||
)
|
||
|
||
logger.info(
|
||
"离线诊断落库完成：baseline={} fingerprint={} "
|
||
"T2={} T1={} T0(infra)={} uncertain(degraded)={} 共 {} 行。",
|
||
baseline_run_id,
|
||
diag_fingerprint,
|
||
counts["T2"],
|
||
counts["T1"],
|
||
counts["T0"],
|
||
counts["uncertain"],
|
||
sum(counts.values()),
|
||
)
|
||
|
||
|
||
def _project_and_persist(
|
||
*,
|
||
result: DiagnosisResult,
|
||
baseline_run_id: str,
|
||
diag_fingerprint: str,
|
||
questions: dict[str, GeneratedQuestion],
|
||
store: DiagnosisSignalStore,
|
||
) -> dict[str, int]:
|
||
"""把 DiagnosisResult 三类产物投影为信号行并逐行 upsert，返回各 tier 计数。
|
||
|
||
参数:
|
||
result: run_diagnosis 的返回，含 error_attributions/infra/degraded 三类产物。
|
||
baseline_run_id: 信号行主键之一。
|
||
diag_fingerprint: 信号行主键之一。
|
||
questions: question_id → GeneratedQuestion，用于取 video_id/task_type。
|
||
store: 诊断信号存储端口。
|
||
|
||
返回:
|
||
{tier: 行数} 计数字典（T2/T1/T0/uncertain），供上层日志与 manifest。
|
||
|
||
关键实现:
|
||
本函数在 run 末（Phase 3）逐行 upsert（单行单事务）；诊断已在 Phase 2 全部
|
||
跑完，故本阶段中途崩溃丢本次 run 未落库的余下行。三桶**非互斥**：
|
||
同一 degraded 错题可能同时出现在 error_attributions（judge 解析失败仍建
|
||
attribution）里，故按 **degraded > infra > attribution** 优先级去重——先落
|
||
degraded/infra，再在 attribution 循环跳过已落题，保证**每题恰写一行、
|
||
counts 恰计一次**（否则同 PK 覆盖会导致 counts 双计且分层错乱）。
|
||
"""
|
||
counts = {"T2": 0, "T1": 0, "T0": 0, "uncertain": 0}
|
||
# 优先级去重：degraded > infra > attribution。先记录高优先集合，
|
||
# attribution 循环遇到已落题即跳过，确保每题唯一落库。
|
||
persisted: set[str] = set()
|
||
|
||
# degraded_question_ids（最高优先）：judge 解析失败降级 → uncertain，信号不可信排除出 T2
|
||
for qid in result.degraded_question_ids:
|
||
q = questions[qid]
|
||
store.upsert(
|
||
DiagnosisSignalRow(
|
||
question_id=qid,
|
||
video_id=q.video_id,
|
||
baseline_run_id=baseline_run_id,
|
||
diag_fingerprint=diag_fingerprint,
|
||
task_type=q.task_type,
|
||
error_type=None,
|
||
cause_category=None,
|
||
tier="uncertain",
|
||
evolution_target=None,
|
||
degraded=True,
|
||
infra=False,
|
||
session_id=None,
|
||
)
|
||
)
|
||
persisted.add(qid)
|
||
counts["uncertain"] += 1
|
||
|
||
# infra_question_ids：基础设施失败护栏排除 → T0，不参与训练主体
|
||
# （防御性跳过已落 degraded 题，虽 infra 通常已在诊断前过滤不重叠）
|
||
for qid in result.infra_question_ids:
|
||
if qid in persisted:
|
||
continue
|
||
q = questions[qid]
|
||
store.upsert(
|
||
DiagnosisSignalRow(
|
||
question_id=qid,
|
||
video_id=q.video_id,
|
||
baseline_run_id=baseline_run_id,
|
||
diag_fingerprint=diag_fingerprint,
|
||
task_type=q.task_type,
|
||
error_type=None,
|
||
cause_category=None,
|
||
tier="T0",
|
||
evolution_target=None,
|
||
degraded=False,
|
||
infra=True,
|
||
session_id=None,
|
||
)
|
||
)
|
||
persisted.add(qid)
|
||
counts["T0"] += 1
|
||
|
||
# error_attributions（最低优先）：defect→T2 / lapse→T1 / 其它→uncertain（由 score_signal 判定）
|
||
# 跳过已作为 degraded/infra 落库的题，避免同 PK 覆盖与 counts 双计。
|
||
for ea in result.error_attributions:
|
||
if ea.question_id in persisted:
|
||
continue
|
||
q = questions[ea.question_id]
|
||
tier = score_signal(cause_category=ea.cause_category, infra=False, degraded=False).tier
|
||
# error_type 是 ErrorAttribution 必填字段（永远已知），确定性派生进化目标。
|
||
evolution_target = evolution_target_of(ea.error_type)
|
||
store.upsert(
|
||
DiagnosisSignalRow(
|
||
question_id=ea.question_id,
|
||
video_id=q.video_id,
|
||
baseline_run_id=baseline_run_id,
|
||
diag_fingerprint=diag_fingerprint,
|
||
task_type=q.task_type,
|
||
error_type=ea.error_type,
|
||
cause_category=ea.cause_category,
|
||
tier=tier,
|
||
evolution_target=evolution_target,
|
||
degraded=False,
|
||
infra=False,
|
||
session_id=None,
|
||
)
|
||
)
|
||
persisted.add(ea.question_id)
|
||
counts[tier] = counts.get(tier, 0) + 1
|
||
|
||
return counts
|