fix: Codex 全量审查修正
Critical: - C1: assemble_mode 'plain' → 'ids'(合法枚举值) - C2: question_id 加入 task_type slug 避免跨题型冲突 Important/Minor: - generate_one 移除未用的 embed_fn/similarity_threshold 参数 - config.py 注释 11→12 同步 - 测试 question_id 断言更新 Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -267,7 +267,7 @@ def _validate_minibatch(config: RunConfig) -> None:
|
|||||||
|
|
||||||
关键实现细节:
|
关键实现细节:
|
||||||
val_size 必须 >= eval_min_per_class * _VIDEO_MME_TASK_TYPE_COUNT,保证验证池
|
val_size 必须 >= eval_min_per_class * _VIDEO_MME_TASK_TYPE_COUNT,保证验证池
|
||||||
能为 Video-MME 的全部 11 个任务类型各保底 eval_min_per_class 题。
|
能为 Video-MME 的全部 12 个任务类型各保底 eval_min_per_class 题。
|
||||||
"""
|
"""
|
||||||
if config.batch_size <= 0:
|
if config.batch_size <= 0:
|
||||||
raise ValueError(f"batch_size 必须 > 0,实际: {config.batch_size}")
|
raise ValueError(f"batch_size 必须 > 0,实际: {config.batch_size}")
|
||||||
|
|||||||
@@ -624,7 +624,7 @@ def parse_vlm_response(
|
|||||||
raise ValueError(f"answer 必须是 A/B/C/D 之一,实际 '{answer}': {raw[:200]}")
|
raise ValueError(f"answer 必须是 A/B/C/D 之一,实际 '{answer}': {raw[:200]}")
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"question_id": f"gen-{video_id}-{seq:03d}",
|
"question_id": f"gen-{video_id}-{task_type.lower().replace(' ', '_')}-{seq:03d}",
|
||||||
"question": data["question"],
|
"question": data["question"],
|
||||||
"options": list(options),
|
"options": list(options),
|
||||||
"answer": answer,
|
"answer": answer,
|
||||||
@@ -669,7 +669,6 @@ def is_duplicate(
|
|||||||
|
|
||||||
async def generate_one(
|
async def generate_one(
|
||||||
vlm: VLMProvider,
|
vlm: VLMProvider,
|
||||||
embed_fn: Callable[[str | list[str]], np.ndarray],
|
|
||||||
tree: TreeIndex,
|
tree: TreeIndex,
|
||||||
video_id: str,
|
video_id: str,
|
||||||
task_type: str,
|
task_type: str,
|
||||||
@@ -678,7 +677,6 @@ async def generate_one(
|
|||||||
exemplars: list[GeneratedQuestion],
|
exemplars: list[GeneratedQuestion],
|
||||||
used_node_ids: set[str],
|
used_node_ids: set[str],
|
||||||
max_retries: int,
|
max_retries: int,
|
||||||
similarity_threshold: float,
|
|
||||||
rng: random.Random,
|
rng: random.Random,
|
||||||
session_id: str,
|
session_id: str,
|
||||||
) -> GeneratedQuestion | None:
|
) -> GeneratedQuestion | None:
|
||||||
@@ -695,7 +693,6 @@ async def generate_one(
|
|||||||
|
|
||||||
参数:
|
参数:
|
||||||
vlm: VLM 调用端口。
|
vlm: VLM 调用端口。
|
||||||
embed_fn: 文本嵌入函数(本函数内未使用,由调用方统一去重)。
|
|
||||||
tree: 三层树索引。
|
tree: 三层树索引。
|
||||||
video_id: 所属视频标识。
|
video_id: 所属视频标识。
|
||||||
task_type: 12 种 Video-MME 题型之一。
|
task_type: 12 种 Video-MME 题型之一。
|
||||||
@@ -703,7 +700,6 @@ async def generate_one(
|
|||||||
exemplars: 少样本示例列表。
|
exemplars: 少样本示例列表。
|
||||||
used_node_ids: 已用节点 ID 集合。
|
used_node_ids: 已用节点 ID 集合。
|
||||||
max_retries: 最大重试次数。
|
max_retries: 最大重试次数。
|
||||||
similarity_threshold: 余弦相似度阈值(本函数内未使用)。
|
|
||||||
rng: 可控随机数生成器。
|
rng: 可控随机数生成器。
|
||||||
session_id: 会话 ID(传递给 VLM 遥测)。
|
session_id: 会话 ID(传递给 VLM 遥测)。
|
||||||
|
|
||||||
|
|||||||
@@ -346,14 +346,14 @@ class TestParseVlmResponse:
|
|||||||
assert result["question"] == "What?"
|
assert result["question"] == "What?"
|
||||||
assert result["answer"] == "A"
|
assert result["answer"] == "A"
|
||||||
assert len(result["options"]) == 4
|
assert len(result["options"]) == 4
|
||||||
assert result["question_id"] == "gen-vid1-001"
|
assert result["question_id"] == "gen-vid1-object_recognition-001"
|
||||||
|
|
||||||
def test_json_in_code_block(self) -> None:
|
def test_json_in_code_block(self) -> None:
|
||||||
"""从 markdown 代码块中提取 JSON。"""
|
"""从 markdown 代码块中提取 JSON。"""
|
||||||
raw = '```json\n{"question": "Q?", "options": ["A. 1", "B. 2", "C. 3", "D. 4"], "answer": "B"}\n```'
|
raw = '```json\n{"question": "Q?", "options": ["A. 1", "B. 2", "C. 3", "D. 4"], "answer": "B"}\n```'
|
||||||
result = parse_vlm_response(raw, "vid1", "Object Recognition", 2)
|
result = parse_vlm_response(raw, "vid1", "Object Recognition", 2)
|
||||||
assert result["question"] == "Q?"
|
assert result["question"] == "Q?"
|
||||||
assert result["question_id"] == "gen-vid1-002"
|
assert result["question_id"] == "gen-vid1-object_recognition-002"
|
||||||
|
|
||||||
def test_invalid_json_raises(self) -> None:
|
def test_invalid_json_raises(self) -> None:
|
||||||
"""非 JSON 文本应抛出 ValueError。"""
|
"""非 JSON 文本应抛出 ValueError。"""
|
||||||
@@ -382,7 +382,7 @@ class TestParseVlmResponse:
|
|||||||
"""seq 应按 3 位零填充格式化到 question_id 中。"""
|
"""seq 应按 3 位零填充格式化到 question_id 中。"""
|
||||||
raw = '{"question": "Q?", "options": ["A. 1", "B. 2", "C. 3", "D. 4"], "answer": "C"}'
|
raw = '{"question": "Q?", "options": ["A. 1", "B. 2", "C. 3", "D. 4"], "answer": "C"}'
|
||||||
result = parse_vlm_response(raw, "video_abc", "Action Reasoning", 42)
|
result = parse_vlm_response(raw, "video_abc", "Action Reasoning", 42)
|
||||||
assert result["question_id"] == "gen-video_abc-042"
|
assert result["question_id"] == "gen-video_abc-action_reasoning-042"
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -448,14 +448,9 @@ class TestGenerateOne:
|
|||||||
content='{"question":"Q?","options":["A. 1","B. 2","C. 3","D. 4"],"answer":"A"}',
|
content='{"question":"Q?","options":["A. 1","B. 2","C. 3","D. 4"],"answer":"A"}',
|
||||||
)
|
)
|
||||||
|
|
||||||
def embed_fn(t: str | list[str]) -> np.ndarray:
|
|
||||||
shape = (1, 4) if isinstance(t, str) else (len(t), 4)
|
|
||||||
return np.zeros(shape, dtype=np.float32)
|
|
||||||
|
|
||||||
tree, vid = self._load_test_tree()
|
tree, vid = self._load_test_tree()
|
||||||
result = await generate_one(
|
result = await generate_one(
|
||||||
vlm=vlm,
|
vlm=vlm,
|
||||||
embed_fn=embed_fn,
|
|
||||||
tree=tree,
|
tree=tree,
|
||||||
video_id=vid,
|
video_id=vid,
|
||||||
task_type="Object Recognition",
|
task_type="Object Recognition",
|
||||||
@@ -463,12 +458,11 @@ class TestGenerateOne:
|
|||||||
exemplars=[],
|
exemplars=[],
|
||||||
used_node_ids=set(),
|
used_node_ids=set(),
|
||||||
max_retries=3,
|
max_retries=3,
|
||||||
similarity_threshold=0.85,
|
|
||||||
rng=random.Random(42),
|
rng=random.Random(42),
|
||||||
session_id="test",
|
session_id="test",
|
||||||
)
|
)
|
||||||
assert result is not None
|
assert result is not None
|
||||||
assert result.question_id == f"gen-{vid}-001"
|
assert result.question_id == f"gen-{vid}-object_recognition-001"
|
||||||
assert result.task_type == "Object Recognition"
|
assert result.task_type == "Object Recognition"
|
||||||
assert result.source_nodes # non-empty
|
assert result.source_nodes # non-empty
|
||||||
assert result.difficulty == "medium"
|
assert result.difficulty == "medium"
|
||||||
@@ -479,13 +473,9 @@ class TestGenerateOne:
|
|||||||
vlm = AsyncMock()
|
vlm = AsyncMock()
|
||||||
vlm.chat_with_images.return_value = MagicMock(content="invalid")
|
vlm.chat_with_images.return_value = MagicMock(content="invalid")
|
||||||
|
|
||||||
def embed_fn(t: str | list[str]) -> np.ndarray:
|
|
||||||
return np.zeros((1, 4), dtype=np.float32)
|
|
||||||
|
|
||||||
tree, vid = self._load_test_tree()
|
tree, vid = self._load_test_tree()
|
||||||
result = await generate_one(
|
result = await generate_one(
|
||||||
vlm=vlm,
|
vlm=vlm,
|
||||||
embed_fn=embed_fn,
|
|
||||||
tree=tree,
|
tree=tree,
|
||||||
video_id=vid,
|
video_id=vid,
|
||||||
task_type="Object Recognition",
|
task_type="Object Recognition",
|
||||||
@@ -493,7 +483,6 @@ class TestGenerateOne:
|
|||||||
exemplars=[],
|
exemplars=[],
|
||||||
used_node_ids=set(),
|
used_node_ids=set(),
|
||||||
max_retries=2,
|
max_retries=2,
|
||||||
similarity_threshold=0.85,
|
|
||||||
rng=random.Random(42),
|
rng=random.Random(42),
|
||||||
session_id="test",
|
session_id="test",
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -594,7 +594,7 @@ async def _run_inference_for_questions(
|
|||||||
ocr=None,
|
ocr=None,
|
||||||
verify_vision=False,
|
verify_vision=False,
|
||||||
anchor=False,
|
anchor=False,
|
||||||
assemble_mode="plain",
|
assemble_mode="ids",
|
||||||
)
|
)
|
||||||
result = await run_inference(
|
result = await run_inference(
|
||||||
group,
|
group,
|
||||||
@@ -861,7 +861,6 @@ async def _run_generate(args: argparse.Namespace) -> None:
|
|||||||
|
|
||||||
async def _generate_with_sem(
|
async def _generate_with_sem(
|
||||||
vlm_client,
|
vlm_client,
|
||||||
embed_fn_inner,
|
|
||||||
tree,
|
tree,
|
||||||
video_id,
|
video_id,
|
||||||
task_type,
|
task_type,
|
||||||
@@ -870,7 +869,6 @@ async def _run_generate(args: argparse.Namespace) -> None:
|
|||||||
exemplars,
|
exemplars,
|
||||||
used_node_ids,
|
used_node_ids,
|
||||||
max_retries_inner,
|
max_retries_inner,
|
||||||
similarity_threshold_inner,
|
|
||||||
rng_inner,
|
rng_inner,
|
||||||
session_id,
|
session_id,
|
||||||
):
|
):
|
||||||
@@ -878,7 +876,6 @@ async def _run_generate(args: argparse.Namespace) -> None:
|
|||||||
async with sem:
|
async with sem:
|
||||||
return await generate_one(
|
return await generate_one(
|
||||||
vlm_client,
|
vlm_client,
|
||||||
embed_fn_inner,
|
|
||||||
tree,
|
tree,
|
||||||
video_id,
|
video_id,
|
||||||
task_type,
|
task_type,
|
||||||
@@ -886,7 +883,6 @@ async def _run_generate(args: argparse.Namespace) -> None:
|
|||||||
exemplars=exemplars,
|
exemplars=exemplars,
|
||||||
used_node_ids=used_node_ids,
|
used_node_ids=used_node_ids,
|
||||||
max_retries=max_retries_inner,
|
max_retries=max_retries_inner,
|
||||||
similarity_threshold=similarity_threshold_inner,
|
|
||||||
rng=rng_inner,
|
rng=rng_inner,
|
||||||
session_id=session_id,
|
session_id=session_id,
|
||||||
)
|
)
|
||||||
@@ -929,7 +925,6 @@ async def _run_generate(args: argparse.Namespace) -> None:
|
|||||||
for _attempt in range(max_retries):
|
for _attempt in range(max_retries):
|
||||||
candidate = await _generate_with_sem(
|
candidate = await _generate_with_sem(
|
||||||
vlm,
|
vlm,
|
||||||
embed_fn,
|
|
||||||
tree,
|
tree,
|
||||||
video_id,
|
video_id,
|
||||||
task_type,
|
task_type,
|
||||||
@@ -937,7 +932,6 @@ async def _run_generate(args: argparse.Namespace) -> None:
|
|||||||
exemplars=exemplars,
|
exemplars=exemplars,
|
||||||
used_node_ids=used_node_ids,
|
used_node_ids=used_node_ids,
|
||||||
max_retries_inner=1,
|
max_retries_inner=1,
|
||||||
similarity_threshold_inner=similarity_threshold,
|
|
||||||
rng_inner=rng,
|
rng_inner=rng,
|
||||||
session_id=session_id,
|
session_id=session_id,
|
||||||
)
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user