iomgaa/Video-Tree-TRM5
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: aggregate inference by question unit with pair AND

Browse Source 
Reuse build_units/unit_correctness (pair contract single entry) in the
inference aggregation step: single questions count as one unit, AR pairs
collapse original+mirror into one unit scored by bidirectional AND. total/
correct/per_task_type are unit-grained; orphan pairs (missing one side) are
warned and dropped, not counted. Per-question predictions still land row by
row (traceability unchanged).
This commit is contained in:
iomgaa
2026-07-15 06:57:48 -04:00
parent 2429dad393
commit 730caa7e9a
3 changed files with 539 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/test_harness_inference.py
+113 -71
View File
@@ -174,13 +174,37 @@ class TestToTextField:
assert "\\u" not in result
def _single_record(
question_id: str,
*,
prediction: str | None,
answer: str,
task_type: str,
steps_used: int,
prompt_tokens: int,
completion_tokens: int,
stop_reason: str,
) -> dict[str, Any]:
"""构造一条 single 题的 prediction record（含 question_id 供 unit 聚合）。"""
return {
"question_id": question_id,
"prediction": prediction,
"answer": answer,
"task_type": task_type,
"steps_used": steps_used,
"prompt_tokens": prompt_tokens,
"completion_tokens": completion_tokens,
"stop_reason": stop_reason,
}
class TestAggregateResults:
"""_aggregate_results 内存聚合测试。"""
"""_aggregate_results unit 粒度聚合测试（single 题：unit 数 = 题数）。"""
@pytest.mark.asyncio
async def test_empty_records(self) -> None:
"""空列表返回零值 InferenceResult。"""
result = _aggregate_results([], "run-empty")
result = _aggregate_results([], [], "run-empty")
assert result.run_id == "run-empty"
assert result.accuracy == 0.0
assert result.total == 0
@@ -193,18 +217,20 @@ class TestAggregateResults:
@pytest.mark.asyncio
async def test_single_correct(self) -> None:
"""单条正确记录 → accuracy=1.0。"""
questions = [_make_question(question_id="q1", task_type="AR", answer="B")]
records = [
{
"prediction": "B",
"answer": "B",
"task_type": "AR",
"steps_used": 3,
"prompt_tokens": 100,
"completion_tokens": 50,
"stop_reason": "finished",
}
_single_record(
"q1",
prediction="B",
answer="B",
task_type="AR",
steps_used=3,
prompt_tokens=100,
completion_tokens=50,
stop_reason="finished",
)
]
result = _aggregate_results(records, "run-1")
result = _aggregate_results(records, questions, "run-1")
assert result.accuracy == 1.0
assert result.total == 1
assert result.correct == 1
@@ -213,36 +239,44 @@ class TestAggregateResults:
@pytest.mark.asyncio
async def test_mixed_correct_wrong(self) -> None:
"""混合正确/错误 → 准确率与步数均正确聚合。"""
records = [
{
"prediction": "B",
"answer": "B",
"task_type": "AR",
"steps_used": 2,
"prompt_tokens": 100,
"completion_tokens": 50,
"stop_reason": "finished",
},
{
"prediction": "C",
"answer": "A",
"task_type": "AR",
"steps_used": 4,
"prompt_tokens": 200,
"completion_tokens": 100,
"stop_reason": "budget_exceeded",
},
{
"prediction": "D",
"answer": "D",
"task_type": "SP",
"steps_used": 1,
"prompt_tokens": 50,
"completion_tokens": 25,
"stop_reason": "finished",
},
questions = [
_make_question(question_id="q1", task_type="AR", answer="B"),
_make_question(question_id="q2", task_type="AR", answer="A"),
_make_question(question_id="q3", task_type="SP", answer="D"),
]
result = _aggregate_results(records, "run-mix")
records = [
_single_record(
"q1",
prediction="B",
answer="B",
task_type="AR",
steps_used=2,
prompt_tokens=100,
completion_tokens=50,
stop_reason="finished",
),
_single_record(
"q2",
prediction="C",
answer="A",
task_type="AR",
steps_used=4,
prompt_tokens=200,
completion_tokens=100,
stop_reason="budget_exceeded",
),
_single_record(
"q3",
prediction="D",
answer="D",
task_type="SP",
steps_used=1,
prompt_tokens=50,
completion_tokens=25,
stop_reason="finished",
),
]
result = _aggregate_results(records, questions, "run-mix")
assert result.total == 3
assert result.correct == 2
assert abs(result.accuracy - 2 / 3) < 1e-9
@@ -252,37 +286,45 @@ class TestAggregateResults:
@pytest.mark.asyncio
async def test_per_task_type_grouping(self) -> None:
"""按 task_type 分组聚合。"""
records = [
{
"prediction": "B",
"answer": "B",
"task_type": "AR",
"steps_used": 1,
"prompt_tokens": 10,
"completion_tokens": 5,
"stop_reason": "finished",
},
{
"prediction": "A",
"answer": "C",
"task_type": "AR",
"steps_used": 2,
"prompt_tokens": 20,
"completion_tokens": 10,
"stop_reason": "finished",
},
{
"prediction": "D",
"answer": "D",
"task_type": "SP",
"steps_used": 3,
"prompt_tokens": 30,
"completion_tokens": 15,
"stop_reason": "finished",
},
"""按 task_type 分组聚合（unit 粒度）。"""
questions = [
_make_question(question_id="q1", task_type="AR", answer="B"),
_make_question(question_id="q2", task_type="AR", answer="C"),
_make_question(question_id="q3", task_type="SP", answer="D"),
]
result = _aggregate_results(records, "run-task")
records = [
_single_record(
"q1",
prediction="B",
answer="B",
task_type="AR",
steps_used=1,
prompt_tokens=10,
completion_tokens=5,
stop_reason="finished",
),
_single_record(
"q2",
prediction="A",
answer="C",
task_type="AR",
steps_used=2,
prompt_tokens=20,
completion_tokens=10,
stop_reason="finished",
),
_single_record(
"q3",
prediction="D",
answer="D",
task_type="SP",
steps_used=3,
prompt_tokens=30,
completion_tokens=15,
stop_reason="finished",
),
]
result = _aggregate_results(records, questions, "run-task")
assert "AR" in result.per_task_type
assert "SP" in result.per_task_type
assert result.per_task_type["AR"]["total"] == 2