"""soak harness 纯函数层测试(M2 设计 §8;T10/T11)。 语料还原器与不变量断言函数是纯函数,用真实 schema 的二次构造微型库测试 (真实场景优先: schema 与 data/soak 实库逐字一致)。 """ import json import sqlite3 import pytest from tools.soak.corpus import ( assemble_frame_messages, chs_image_messages, load_replay_payloads, load_trace_chains, ) from tools.soak.scenarios import weighted_mix def _mini_harness_db(path): """predictions 表 schema 与 data/soak/harness.db 逐字一致的微型库。""" conn = sqlite3.connect(path) conn.execute( "CREATE TABLE predictions (run_id TEXT, timestamp TEXT, video_id TEXT," " question_id TEXT, task_type TEXT, prediction TEXT, answer TEXT, evidence TEXT," " reasoning TEXT, steps_used INTEGER, prompt_tokens INTEGER," " completion_tokens INTEGER, stop_reason TEXT, steps_json TEXT)" ) steps = [ { "thought": f"思考{i}", "tool_call": {"tool": "view_node", "args": {"i": i}}, "tool_output": f"输出{i}" * 50, } for i in range(3) ] conn.execute( "INSERT INTO predictions VALUES ('r1','t','vid1','q1','synopsis','A','A','','',3,1,1,'stop',?)", (json.dumps(steps, ensure_ascii=False),), ) conn.execute( "INSERT INTO predictions VALUES ('r1','t','vid2','q2','synopsis','B','B','','',0,1,1,'stop','[]')" ) conn.commit() conn.close() def _mini_telemetry_db(path): conn = sqlite3.connect(path) conn.execute( "CREATE TABLE llm_calls (call_id TEXT PRIMARY KEY, parent_call_id TEXT," " session_id TEXT, model_name TEXT, provider TEXT, messages TEXT, response TEXT," " thinking TEXT, prompt_tokens INTEGER, completion_tokens INTEGER, latency_ms INTEGER," " ttft_ms REAL, max_inter_token_ms REAL, cache_hit INTEGER, error TEXT, created_at TEXT)" ) msgs = json.dumps([{"role": "user", "content": "回放负载"}]) conn.execute( "INSERT INTO llm_calls VALUES ('c1',NULL,'s1','m','p',?,'r','',1,1,1,NULL,NULL,0,NULL,'t')", (msgs,), ) conn.execute( "INSERT INTO llm_calls VALUES ('c2',NULL,'s1','m','p','not-json','r','',1,1,1,NULL,NULL,0,NULL,'t')" ) conn.commit() conn.close() class TestTraceChains: def test_cumulative_chain_shapes(self, tmp_path): db = tmp_path / "h.db" _mini_harness_db(db) chains = load_trace_chains(db) assert len(chains) == 1 # steps_used=0 的行剔除 chain = chains[0] assert len(chain) == 3 # 每步一个累积快照 # 快照单调增长: system+user 起步,每步追加 assistant+user assert [len(snap) for snap in chain] == [4, 6, 8] first = chain[0] assert first[0]["role"] == "system" and first[1]["role"] == "user" assert first[2]["role"] == "assistant" and "思考0" in first[2]["content"] assert first[3]["role"] == "user" and "输出0" in first[3]["content"] # 后续快照是前缀扩展(累积语义) assert chain[1][:4] == chain[0] def test_max_steps_cap(self, tmp_path): db = tmp_path / "h.db" _mini_harness_db(db) assert [len(s) for s in load_trace_chains(db, max_steps=2)[0]] == [4, 6] class TestReplayPayloads: def test_loads_and_skips_unparseable(self, tmp_path): db = tmp_path / "t.db" _mini_telemetry_db(db) payloads = load_replay_payloads(db) assert len(payloads) == 1 # 坏 JSON 行剔除 assert payloads[0][0]["content"] == "回放负载" class TestImageAssembly: def test_frame_messages_two_tiers(self, tmp_path): vid = tmp_path / "vid1" / "frames" vid.mkdir(parents=True) for i in range(6): (vid / f"f{i}.jpg").write_bytes(b"\xff\xd8fakejpeg" + bytes([i])) msgs = assemble_frame_messages(tmp_path, n_frames=4, rng=lambda: 0.0) parts = msgs[0]["content"] images = [p for p in parts if p["type"] == "image_url"] assert len(images) == 4 assert images[0]["image_url"]["url"].startswith("data:image/jpeg;base64,") def test_chs_single_image_with_instruction(self, tmp_path): (tmp_path / "chs_0001.jpg").write_bytes(b"\xff\xd8fake") msgs = chs_image_messages(tmp_path / "chs_0001.jpg") parts = msgs[0]["content"] assert any(p["type"] == "image_url" for p in parts) text = next(p["text"] for p in parts if p["type"] == "text") assert "JSON" in text # 固定结构化指令 class TestWeightedMix: def test_ratio_approximation(self): import random rng = random.Random(42).random weights = {"P1": 0.1, "P2": 0.2, "P3": 0.5, "P5": 0.2} picks = [weighted_mix(weights, rng) for _ in range(10_000)] for key, w in weights.items(): assert picks.count(key) / 10_000 == pytest.approx(w, abs=0.05) def test_invalid_weights_rejected(self): with pytest.raises(ValueError): weighted_mix({"P1": 0.0}, lambda: 0.5) # ═══════════ T11: 记分板硬不变量(纯函数) ═══════════ from tools.soak.scoreboard import ( # noqa: E402 inv_call_ids_unique, inv_rows_match_calls, inv_rpm_never_exceeded, inv_rss_stable, structured_success_rate, ) def _row(call_id, *, source="s1", created_at="2026-07-20T10:00:00", error=None, session="r-p3"): return { "call_id": call_id, "source_name": source, "created_at": created_at, "error": error, "session_id": session, } class TestInvariants: def test_rows_match_calls(self): rows = [_row("a"), _row("b")] inv_rows_match_calls(rows, expected_calls=2) with pytest.raises(AssertionError): inv_rows_match_calls(rows, expected_calls=3) def test_call_ids_unique(self): inv_call_ids_unique([_row("a"), _row("b")]) with pytest.raises(AssertionError): inv_call_ids_unique([_row("a"), _row("a")]) def test_rpm_minute_bucket(self): # 固定分钟窗口口径(与限流器 int(sec/60) 同源;滑动窗会对合法的 # 跨窗背靠背流量误报,不采用) rows = [ _row("a", created_at="2026-07-20T10:00:01"), _row("b", created_at="2026-07-20T10:00:59"), _row("c", created_at="2026-07-20T10:01:01"), ] inv_rpm_never_exceeded(rows, {"s1": 2}) rows.append(_row("d", created_at="2026-07-20T10:00:30")) with pytest.raises(AssertionError): inv_rpm_never_exceeded(rows, {"s1": 2}) def test_rpm_ignores_unlimited_sources(self): inv_rpm_never_exceeded([_row(str(i)) for i in range(100)], {"s1": 0}) def test_rss_stable(self): inv_rss_stable([100.0, 105.0, 110.0], max_growth_mb=50.0) with pytest.raises(AssertionError): inv_rss_stable([100.0, 400.0], max_growth_mb=50.0) def test_structured_success_rate(self): rows = [ _row("a"), _row("b", error="ResultInvalidError: x"), _row("c", session="r-p1"), # 非 P3 剔除 ] assert structured_success_rate(rows, session_suffix="-p3") == pytest.approx(0.5)