fix(soak): 给 plan 与 execute 补上完成通路,它们原本必然跑满预算
打真实模型跑通一个完整任务时发现的:这两个阶段的工具集里没有带完成标记的工具,解释器
也从不产出最终回答,于是模型在第 4 步写完产出物之后还在继续读文档,直到撞上步数上限。
按原来的 50/50/16 算,二十个任务要两千三百多次调用,而且整批的停止原因会全是
step_budget,别的什么都压不出来。
解释器加一条最终回答支路({"final_answer": "..."}),plan 与 execute 的提示词写死收尾
动作;summarize 不变,仍然只能靠 submit_finding 结束。这样三条完成路径同时在场:模型
自报最终回答、agent 调用带完成标记的工具、环境报告完成(AppWorld 那路),它们的可信度
各不相同,压测正需要这个对照。
预算随之下调到 20/25/16——实测每阶段真实用 5 到 7 步,留了一倍余量。原来那个 50 的来历
是 gov-auditor.yaml 的 turns 上限,但那边靠编排校验产物落盘来收阶段,跑满 turns 无所谓;
这里靠模型自己收尾,上限定高只会让它在产出物写完之后接着白烧。
实测:plan 7 步 agent_finished、execute 6 步 agent_finished、summarize 5 步
task_completed,一个任务 18 次调用。
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -21,7 +21,7 @@ from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
|
||||
from polyloop.ports import Action, InvalidDecision
|
||||
from polyloop.ports import Action, FinalAnswer, InvalidDecision
|
||||
from polyloop.tools import ToolRegistry
|
||||
from polyloop.types import ModelReply, ReplayPolicy
|
||||
from tools.soak.scenarios.govdoc import (
|
||||
@@ -217,7 +217,7 @@ def test_contract_4_action_carries_text():
|
||||
)
|
||||
def test_contract_5_parse_never_raises(content: str):
|
||||
parsed = GovDocParser().parse(_reply(content))
|
||||
assert isinstance(parsed.decision, Action | InvalidDecision)
|
||||
assert isinstance(parsed.decision, Action | FinalAnswer | InvalidDecision)
|
||||
|
||||
|
||||
def test_tolerates_code_fences():
|
||||
@@ -264,6 +264,74 @@ def test_five_parse_failures_get_five_explanations():
|
||||
assert "arguments" in explanations["arguments 不是对象"]
|
||||
|
||||
|
||||
def test_final_answer_branch():
|
||||
parsed = GovDocParser().parse(_reply('{"final_answer": "已写出 plan.md,列了 3 处候选证据。"}'))
|
||||
assert isinstance(parsed.decision, FinalAnswer)
|
||||
assert parsed.decision.text == "已写出 plan.md,列了 3 处候选证据。"
|
||||
assert len(parsed.history_text) <= len(
|
||||
'{"final_answer": "已写出 plan.md,列了 3 处候选证据。"}'
|
||||
)
|
||||
|
||||
|
||||
def test_final_answer_tolerates_code_fences():
|
||||
inner = '{"final_answer": "已写出 evidence.md,3 条证据。"}'
|
||||
for fenced in (
|
||||
f"```json\n{inner}\n```",
|
||||
f"```\n{inner}\n```",
|
||||
f"这一阶段做完了:\n```json\n{inner}\n```\n",
|
||||
):
|
||||
parsed = GovDocParser().parse(_reply(fenced))
|
||||
assert isinstance(parsed.decision, FinalAnswer), fenced
|
||||
assert parsed.decision.text == "已写出 evidence.md,3 条证据。"
|
||||
|
||||
|
||||
def test_tool_wins_when_both_keys_are_present():
|
||||
"""一边调工具一边宣布做完时以工具为准:按 final_answer 收尾会把那次调用整个丢掉。"""
|
||||
parsed = GovDocParser().parse(
|
||||
_reply(
|
||||
'{"tool": "write_note", "arguments": {"filename": "plan.md", "content": "x"},'
|
||||
' "final_answer": "我写完了"}'
|
||||
)
|
||||
)
|
||||
assert isinstance(parsed.decision, Action)
|
||||
assert parsed.decision.tool_call is not None
|
||||
assert parsed.decision.tool_call.name == "write_note"
|
||||
|
||||
|
||||
def test_flattened_arguments_do_not_swallow_final_answer():
|
||||
parsed = GovDocParser().parse(
|
||||
_reply(
|
||||
'{"tool": "write_note", "filename": "plan.md", "content": "x", "final_answer": "完"}'
|
||||
)
|
||||
)
|
||||
assert isinstance(parsed.decision, Action)
|
||||
assert parsed.decision.tool_call is not None
|
||||
assert parsed.decision.tool_call.arguments == {"filename": "plan.md", "content": "x"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"content",
|
||||
[
|
||||
'{"answer": "我做完了"}',
|
||||
'{"arguments": {"path": "tender.md"}}',
|
||||
'{"final_answer": ""}',
|
||||
'{"final_answer": " "}',
|
||||
"{}",
|
||||
],
|
||||
)
|
||||
def test_neither_key_is_still_invalid(content: str):
|
||||
decision = GovDocParser().parse(_reply(content)).decision
|
||||
assert isinstance(decision, InvalidDecision)
|
||||
assert decision.explanation.strip()
|
||||
|
||||
|
||||
def test_missing_key_explanation_names_both_shapes():
|
||||
decision = GovDocParser().parse(_reply('{"note": "x"}')).decision
|
||||
assert isinstance(decision, InvalidDecision)
|
||||
assert "tool" in decision.explanation
|
||||
assert "final_answer" in decision.explanation
|
||||
|
||||
|
||||
def test_parser_reports_its_parameters():
|
||||
assert GovDocParser().parameters() == {"kind": "govdoc_json_tool_call"}
|
||||
|
||||
@@ -478,7 +546,11 @@ def test_all_three_phases_assemble(tmp_path: Path):
|
||||
assert request.injections == {}
|
||||
|
||||
|
||||
def test_budgets_follow_gov_auditor_turn_limits(tmp_path: Path):
|
||||
def test_budgets_are_the_measured_step_counts(tmp_path: Path):
|
||||
"""20 / 25 / 16 是实测出来的,不是照抄 gov-auditor.yaml 的 50 / 50 / 16。
|
||||
|
||||
断言具体数字是因为这三个数直接决定一次全量压测的调用量,改动必须是有意的。
|
||||
"""
|
||||
task = _sample_task()
|
||||
budgets = {
|
||||
phase: build_run_request(
|
||||
@@ -490,12 +562,35 @@ def test_budgets_follow_gov_auditor_turn_limits(tmp_path: Path):
|
||||
).budget
|
||||
for phase in ("plan", "execute", "summarize")
|
||||
}
|
||||
assert (budgets["plan"].max_steps, budgets["plan"].max_actions) == (50, 50)
|
||||
assert (budgets["execute"].max_steps, budgets["execute"].max_actions) == (50, 50)
|
||||
assert (budgets["plan"].max_steps, budgets["plan"].max_actions) == (20, 20)
|
||||
assert (budgets["execute"].max_steps, budgets["execute"].max_actions) == (25, 25)
|
||||
assert (budgets["summarize"].max_steps, budgets["summarize"].max_actions) == (16, 16)
|
||||
for budget in budgets.values():
|
||||
assert budget.max_consecutive_parse_failures == 3
|
||||
assert budget.max_prompt_chars == 400_000
|
||||
# 一次全量(20 个任务 × 3 个阶段)的步数上限。这一条守的是额度,不是行为。
|
||||
assert sum(budget.max_steps for budget in budgets.values()) * 20 == 1220
|
||||
|
||||
|
||||
def test_plan_and_execute_prompts_spell_out_the_final_answer_closing(tmp_path: Path):
|
||||
"""plan 与 execute 的工具集里没有带 completes_run 的工具,收尾只能靠最终回答。
|
||||
|
||||
提示词不写清楚这一步,这两个阶段就必然跑满预算——实测过,停止原因全是 step_budget。
|
||||
"""
|
||||
task = _sample_task()
|
||||
registry = _tools(tmp_path).registry()
|
||||
for phase in ("plan", "execute"):
|
||||
narrowed = registry.restrict_to(PHASE_TOOLS[phase])
|
||||
system = build_context(task=task, phase=phase, tools=narrowed).run_level[0].content[0].text
|
||||
assert '{"final_answer":' in system
|
||||
assert not any(narrowed.spec_for(name).completes_run for name in narrowed.names())
|
||||
# summarize 不变:它靠 submit_finding 这条提交型完成通路结束,不给最终回答这条路。
|
||||
summarize = registry.restrict_to(PHASE_TOOLS["summarize"])
|
||||
system = (
|
||||
build_context(task=task, phase="summarize", tools=summarize).run_level[0].content[0].text
|
||||
)
|
||||
assert "final_answer" not in system
|
||||
assert any(summarize.spec_for(name).completes_run for name in summarize.names())
|
||||
|
||||
|
||||
def test_context_carries_tools_but_not_the_document_body(tmp_path: Path):
|
||||
|
||||
Reference in New Issue
Block a user