fix(soak): 给 plan 与 execute 补上完成通路,它们原本必然跑满预算

打真实模型跑通一个完整任务时发现的:这两个阶段的工具集里没有带完成标记的工具,解释器
也从不产出最终回答,于是模型在第 4 步写完产出物之后还在继续读文档,直到撞上步数上限。
按原来的 50/50/16 算,二十个任务要两千三百多次调用,而且整批的停止原因会全是
step_budget,别的什么都压不出来。

解释器加一条最终回答支路({"final_answer": "..."}),plan 与 execute 的提示词写死收尾
动作;summarize 不变,仍然只能靠 submit_finding 结束。这样三条完成路径同时在场:模型
自报最终回答、agent 调用带完成标记的工具、环境报告完成(AppWorld 那路),它们的可信度
各不相同,压测正需要这个对照。

预算随之下调到 20/25/16——实测每阶段真实用 5 到 7 步,留了一倍余量。原来那个 50 的来历
是 gov-auditor.yaml 的 turns 上限,但那边靠编排校验产物落盘来收阶段,跑满 turns 无所谓;
这里靠模型自己收尾,上限定高只会让它在产出物写完之后接着白烧。

实测:plan 7 步 agent_finished、execute 6 步 agent_finished、summarize 5 步
task_completed,一个任务 18 次调用。

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-08-11 08:58:57 -04:00
parent f277197071
commit 10f4a49f37
2 changed files with 177 additions and 29 deletions
+100 -5
View File
@@ -21,7 +21,7 @@ from typing import TYPE_CHECKING
import pytest
from polyloop.ports import Action, InvalidDecision
from polyloop.ports import Action, FinalAnswer, InvalidDecision
from polyloop.tools import ToolRegistry
from polyloop.types import ModelReply, ReplayPolicy
from tools.soak.scenarios.govdoc import (
@@ -217,7 +217,7 @@ def test_contract_4_action_carries_text():
)
def test_contract_5_parse_never_raises(content: str):
parsed = GovDocParser().parse(_reply(content))
assert isinstance(parsed.decision, Action | InvalidDecision)
assert isinstance(parsed.decision, Action | FinalAnswer | InvalidDecision)
def test_tolerates_code_fences():
@@ -264,6 +264,74 @@ def test_five_parse_failures_get_five_explanations():
assert "arguments" in explanations["arguments 不是对象"]
def test_final_answer_branch():
parsed = GovDocParser().parse(_reply('{"final_answer": "已写出 plan.md,列了 3 处候选证据。"}'))
assert isinstance(parsed.decision, FinalAnswer)
assert parsed.decision.text == "已写出 plan.md,列了 3 处候选证据。"
assert len(parsed.history_text) <= len(
'{"final_answer": "已写出 plan.md,列了 3 处候选证据。"}'
)
def test_final_answer_tolerates_code_fences():
inner = '{"final_answer": "已写出 evidence.md3 条证据。"}'
for fenced in (
f"```json\n{inner}\n```",
f"```\n{inner}\n```",
f"这一阶段做完了:\n```json\n{inner}\n```\n",
):
parsed = GovDocParser().parse(_reply(fenced))
assert isinstance(parsed.decision, FinalAnswer), fenced
assert parsed.decision.text == "已写出 evidence.md3 条证据。"
def test_tool_wins_when_both_keys_are_present():
"""一边调工具一边宣布做完时以工具为准:按 final_answer 收尾会把那次调用整个丢掉。"""
parsed = GovDocParser().parse(
_reply(
'{"tool": "write_note", "arguments": {"filename": "plan.md", "content": "x"},'
' "final_answer": "我写完了"}'
)
)
assert isinstance(parsed.decision, Action)
assert parsed.decision.tool_call is not None
assert parsed.decision.tool_call.name == "write_note"
def test_flattened_arguments_do_not_swallow_final_answer():
parsed = GovDocParser().parse(
_reply(
'{"tool": "write_note", "filename": "plan.md", "content": "x", "final_answer": ""}'
)
)
assert isinstance(parsed.decision, Action)
assert parsed.decision.tool_call is not None
assert parsed.decision.tool_call.arguments == {"filename": "plan.md", "content": "x"}
@pytest.mark.parametrize(
"content",
[
'{"answer": "我做完了"}',
'{"arguments": {"path": "tender.md"}}',
'{"final_answer": ""}',
'{"final_answer": " "}',
"{}",
],
)
def test_neither_key_is_still_invalid(content: str):
decision = GovDocParser().parse(_reply(content)).decision
assert isinstance(decision, InvalidDecision)
assert decision.explanation.strip()
def test_missing_key_explanation_names_both_shapes():
decision = GovDocParser().parse(_reply('{"note": "x"}')).decision
assert isinstance(decision, InvalidDecision)
assert "tool" in decision.explanation
assert "final_answer" in decision.explanation
def test_parser_reports_its_parameters():
assert GovDocParser().parameters() == {"kind": "govdoc_json_tool_call"}
@@ -478,7 +546,11 @@ def test_all_three_phases_assemble(tmp_path: Path):
assert request.injections == {}
def test_budgets_follow_gov_auditor_turn_limits(tmp_path: Path):
def test_budgets_are_the_measured_step_counts(tmp_path: Path):
"""20 / 25 / 16 是实测出来的,不是照抄 gov-auditor.yaml 的 50 / 50 / 16。
断言具体数字是因为这三个数直接决定一次全量压测的调用量,改动必须是有意的。
"""
task = _sample_task()
budgets = {
phase: build_run_request(
@@ -490,12 +562,35 @@ def test_budgets_follow_gov_auditor_turn_limits(tmp_path: Path):
).budget
for phase in ("plan", "execute", "summarize")
}
assert (budgets["plan"].max_steps, budgets["plan"].max_actions) == (50, 50)
assert (budgets["execute"].max_steps, budgets["execute"].max_actions) == (50, 50)
assert (budgets["plan"].max_steps, budgets["plan"].max_actions) == (20, 20)
assert (budgets["execute"].max_steps, budgets["execute"].max_actions) == (25, 25)
assert (budgets["summarize"].max_steps, budgets["summarize"].max_actions) == (16, 16)
for budget in budgets.values():
assert budget.max_consecutive_parse_failures == 3
assert budget.max_prompt_chars == 400_000
# 一次全量(20 个任务 × 3 个阶段)的步数上限。这一条守的是额度,不是行为。
assert sum(budget.max_steps for budget in budgets.values()) * 20 == 1220
def test_plan_and_execute_prompts_spell_out_the_final_answer_closing(tmp_path: Path):
"""plan 与 execute 的工具集里没有带 completes_run 的工具,收尾只能靠最终回答。
提示词不写清楚这一步,这两个阶段就必然跑满预算——实测过,停止原因全是 step_budget。
"""
task = _sample_task()
registry = _tools(tmp_path).registry()
for phase in ("plan", "execute"):
narrowed = registry.restrict_to(PHASE_TOOLS[phase])
system = build_context(task=task, phase=phase, tools=narrowed).run_level[0].content[0].text
assert '{"final_answer":' in system
assert not any(narrowed.spec_for(name).completes_run for name in narrowed.names())
# summarize 不变:它靠 submit_finding 这条提交型完成通路结束,不给最终回答这条路。
summarize = registry.restrict_to(PHASE_TOOLS["summarize"])
system = (
build_context(task=task, phase="summarize", tools=summarize).run_level[0].content[0].text
)
assert "final_answer" not in system
assert any(summarize.spec_for(name).completes_run for name in summarize.names())
def test_context_carries_tools_but_not_the_document_body(tmp_path: Path):