feat: feed selector health and demote in-call failed sources
RetryMW keeps a per-call failure map (local, never instance state): a source failing twice in one call yields to the next candidate. Attempt outcomes feed OutcomeAwareSelector behind a swallow-and-warn guard; ResultInvalid and provider-rejected paths record success with count_attempt=False so the breaker window stays clean. Same accounting applied in EmbeddingClient.
This commit is contained in:
@@ -97,6 +97,7 @@ def _harness(
|
||||
quota_full="wait",
|
||||
global_limits=_NO_GLOBAL,
|
||||
rng=lambda: 0.0,
|
||||
selector=None,
|
||||
):
|
||||
clock = clock or FakeClock()
|
||||
limiter = InMemoryLimiter(
|
||||
@@ -112,7 +113,7 @@ def _harness(
|
||||
mw = RetryMW(
|
||||
scope="llm",
|
||||
sources=sources,
|
||||
selector=RoundRobinSelector(),
|
||||
selector=selector if selector is not None else RoundRobinSelector(),
|
||||
limiter=limiter,
|
||||
gate=gate,
|
||||
transport=transport,
|
||||
@@ -351,3 +352,94 @@ class TestCancellation:
|
||||
# 探针租约已归还: 下一 caller 立即拿到探针而非等租约过期
|
||||
nxt = await gate.try_enter("a", "w2")
|
||||
assert nxt.allowed and nxt.is_probe
|
||||
|
||||
|
||||
class StaticSelector:
|
||||
"""固定配置序,隔离测试调用内降权(不带 record_outcome)。"""
|
||||
|
||||
def order(self, sources, stats):
|
||||
return list(sources)
|
||||
|
||||
|
||||
class RecordingSelector(StaticSelector):
|
||||
def __init__(self):
|
||||
self.outcomes = []
|
||||
|
||||
def record_outcome(self, source_name, ok):
|
||||
self.outcomes.append((source_name, ok))
|
||||
|
||||
|
||||
class ExplodingSelector(StaticSelector):
|
||||
def record_outcome(self, source_name, ok):
|
||||
raise RuntimeError("sink boom")
|
||||
|
||||
|
||||
class TestM25Orchestration:
|
||||
"""M2.5 设计 §3.3: 调用内失败降权 + 健康喂数(先红后绿)。"""
|
||||
|
||||
async def test_failed_source_demoted_after_two_strikes(self):
|
||||
# 失败 1 次仍首选(原地退避重试);失败 2 次让位次优源
|
||||
mw, _, _, transport, _, _ = _harness(
|
||||
[_src("a"), _src("b")],
|
||||
[TransientError("1"), TransientError("2"), _ok()],
|
||||
selector=StaticSelector(),
|
||||
)
|
||||
resp = await mw(_REQ)
|
||||
assert [n for n, _ in transport.calls] == ["a", "a", "b"]
|
||||
assert resp.source_name == "b"
|
||||
|
||||
async def test_attempt_fails_reset_between_calls(self):
|
||||
mw, _, _, transport, _, _ = _harness(
|
||||
[_src("a"), _src("b")],
|
||||
[TransientError("1"), TransientError("2"), _ok(), _ok()],
|
||||
selector=StaticSelector(),
|
||||
)
|
||||
await mw(_REQ)
|
||||
await mw(_REQ) # 新调用状态清零: 回到首选 a
|
||||
assert [n for n, _ in transport.calls] == ["a", "a", "b", "a"]
|
||||
|
||||
async def test_outcome_feeding_success_and_transient(self):
|
||||
sel = RecordingSelector()
|
||||
mw, _, _, _, _, _ = _harness(
|
||||
[_src("a"), _src("b")], [TransientError("1"), _ok()], selector=sel
|
||||
)
|
||||
await mw(_REQ)
|
||||
assert sel.outcomes == [("a", False), ("a", True)]
|
||||
|
||||
async def test_outcome_feeding_source_dead(self):
|
||||
sel = RecordingSelector()
|
||||
mw, _, _, _, _, _ = _harness(
|
||||
[_src("a"), _src("b")], [SourceDeadError("401"), _ok()], selector=sel
|
||||
)
|
||||
await mw(_REQ)
|
||||
assert sel.outcomes == [("a", False), ("b", True)]
|
||||
|
||||
async def test_result_invalid_and_rejected_not_fed(self):
|
||||
sel = RecordingSelector()
|
||||
mw, _, _, _, _, _ = _harness(
|
||||
[_src("a")], [ResultInvalidError("bad", raw_text="x")], selector=sel
|
||||
)
|
||||
with pytest.raises(ResultInvalidError):
|
||||
await mw(_REQ)
|
||||
sel2 = RecordingSelector()
|
||||
mw2, _, _, _, _, _ = _harness(
|
||||
[_src("a")],
|
||||
[RequestRejectedError("400", source_name="a", status_code=400)],
|
||||
selector=sel2,
|
||||
)
|
||||
with pytest.raises(RequestRejectedError):
|
||||
await mw2(_REQ)
|
||||
assert sel.outcomes == [] and sel2.outcomes == []
|
||||
|
||||
async def test_outcome_sink_exception_swallowed(self):
|
||||
mw, _, _, _, _, _ = _harness([_src("a")], [_ok()], selector=ExplodingSelector())
|
||||
resp = await mw(_REQ)
|
||||
assert resp.content == "ok" # 喂数异常不得打断真实成功返回
|
||||
|
||||
async def test_result_invalid_gate_window_untouched(self):
|
||||
# 坏结果 ≠ 坏服务: count_attempt=False,失败率窗口 attempts 不得增长
|
||||
mw, _, gate, _, _, _ = _harness([_src("a")], [ResultInvalidError("bad", raw_text="x")])
|
||||
with pytest.raises(ResultInvalidError):
|
||||
await mw(_REQ)
|
||||
g = gate._gates["a"]
|
||||
assert g.a0 + g.a1 == 0
|
||||
|
||||
Reference in New Issue
Block a user