feat: 诊断（任务书 18 第一块）：诊断数学、四种提交、一次运行跑多个实验、账本与材料、flow 诊断各步
消融默认关，关掉时解释阶段一并跳过；选诊断实验用信息价值减成本。 eval/diagnosis/research.json 为第三块配置。vocab 字段说明的出口文字随提示词一起改，见下一次提交的 prompts/ 与同文件改动（本次含在内）。 Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -177,6 +177,7 @@ test("写不出来的组合：朴素循环里写开关、规则或它用不到
|
||||
critique: false,
|
||||
firstPrediction: false,
|
||||
roundZero: { enabled: false },
|
||||
diagnosis: { enabled: false },
|
||||
ablation: { enabled: false },
|
||||
batchRule: "uniformRandom",
|
||||
openReserve: 0.1,
|
||||
@@ -200,6 +201,35 @@ test("写不出来的组合：关掉的第零轮写讨论次数、关掉的消
|
||||
assert.match(errorOf(ablation), /^\/flow\/ablation\/cellRule：不认识的字段$/m);
|
||||
});
|
||||
|
||||
/** 诊断开着的完整流程配置。 */
|
||||
function diagnosisConfig(): ReturnType<typeof fullConfig> {
|
||||
const config = fullConfig();
|
||||
(config.flow as Obj).diagnosis = { enabled: true, perResearcher: 2, maxExperiments: 4 };
|
||||
return config;
|
||||
}
|
||||
|
||||
test("诊断：开着的配置通过校验；逐个删字段、写 0、写小数、多写字段都报错；关着的不能写两个参数", () => {
|
||||
assert.deepEqual(checkConfig(structuredClone(diagnosisConfig())), diagnosisConfig());
|
||||
|
||||
for (const key of ["enabled", "perResearcher", "maxExperiments"]) {
|
||||
const config = diagnosisConfig();
|
||||
delete ((config.flow as Obj).diagnosis as Obj)[key];
|
||||
assert.match(errorOf(config), new RegExp(`^/flow/diagnosis/${key}：`, "m"));
|
||||
}
|
||||
const cases: Array<[Obj, RegExp]> = [
|
||||
[{ enabled: true, perResearcher: 0, maxExperiments: 4 }, /^\/flow\/diagnosis\/perResearcher：不能小于 1$/m],
|
||||
[{ enabled: true, perResearcher: 2, maxExperiments: 2.5 }, /^\/flow\/diagnosis\/maxExperiments：应为整数$/m],
|
||||
[{ enabled: true, perResearcher: 2, maxExperiments: 4, rule: "x" }, /^\/flow\/diagnosis\/rule：不认识的字段$/m],
|
||||
[{ enabled: false, perResearcher: 2 }, /^\/flow\/diagnosis\/perResearcher：不认识的字段$/m],
|
||||
[{ enabled: "yes" }, /^\/flow\/diagnosis\/enabled：缺少或取值不对，只能是 false、true 之一$/m],
|
||||
];
|
||||
for (const [diagnosis, expected] of cases) {
|
||||
const config = fullConfig();
|
||||
(config.flow as Obj).diagnosis = diagnosis;
|
||||
assert.match(errorOf(config), expected);
|
||||
}
|
||||
});
|
||||
|
||||
test("写不出来的组合：规则名不在 rules 登记表里", () => {
|
||||
const batch = fullConfig();
|
||||
(batch.flow as Obj).batchRule = "expected-best";
|
||||
|
||||
@@ -29,6 +29,7 @@ export function fullConfig(): ResearchConfig {
|
||||
roundZero: { enabled: true, discussions: 2 },
|
||||
firstPrediction: true,
|
||||
critique: true,
|
||||
diagnosis: { enabled: false },
|
||||
ablation: { enabled: true, cellRule: "valueMinusCost" },
|
||||
batchRule: "bestExpected",
|
||||
openReserve: 0.1,
|
||||
|
||||
@@ -28,7 +28,7 @@ function assertReasons(list: StepList): void {
|
||||
}
|
||||
|
||||
test("步骤清单：完整流程全开", () => {
|
||||
const list = stepList(fullConfig());
|
||||
const list = listOf((f) => (f.diagnosis = { enabled: true, perResearcher: 2, maxExperiments: 4 }));
|
||||
assert.equal(list.preset, "full");
|
||||
assert.deepEqual(rows(list.study), [
|
||||
["freezeConfig", E, null],
|
||||
@@ -44,6 +44,13 @@ test("步骤清单：完整流程全开", () => {
|
||||
assert.deepEqual(rows(list.round), [
|
||||
["openRound", E, null],
|
||||
["carryOverPool", E, null],
|
||||
["diagnosisProposal", E, null],
|
||||
["diagnosisDedupe", E, null],
|
||||
["diagnosisPrediction", E, null],
|
||||
["selectDiagnosis", E, { name: "valueMinusCost", version: 1 }],
|
||||
["diagnosisImplementation", E, null],
|
||||
["diagnosisRun", E, null],
|
||||
["diagnosisSettle", E, null],
|
||||
["proposal", E, null],
|
||||
["dedupe", E, null],
|
||||
["firstPrediction", E, null],
|
||||
@@ -95,6 +102,13 @@ test("步骤清单：只关第一次预测", () => {
|
||||
assert.deepEqual(rows(list.round), [
|
||||
["openRound", E, null],
|
||||
["carryOverPool", E, null],
|
||||
["diagnosisProposal", S, null],
|
||||
["diagnosisDedupe", S, null],
|
||||
["diagnosisPrediction", S, null],
|
||||
["selectDiagnosis", S, null],
|
||||
["diagnosisImplementation", S, null],
|
||||
["diagnosisRun", S, null],
|
||||
["diagnosisSettle", S, null],
|
||||
["proposal", E, null],
|
||||
["dedupe", E, null],
|
||||
["firstPrediction", S, null],
|
||||
@@ -128,6 +142,13 @@ test("步骤清单：只关挑刺与回应，协议照样冻结", () => {
|
||||
assert.deepEqual(rows(list.round), [
|
||||
["openRound", E, null],
|
||||
["carryOverPool", E, null],
|
||||
["diagnosisProposal", S, null],
|
||||
["diagnosisDedupe", S, null],
|
||||
["diagnosisPrediction", S, null],
|
||||
["selectDiagnosis", S, null],
|
||||
["diagnosisImplementation", S, null],
|
||||
["diagnosisRun", S, null],
|
||||
["diagnosisSettle", S, null],
|
||||
["proposal", E, null],
|
||||
["dedupe", E, null],
|
||||
["firstPrediction", E, null],
|
||||
@@ -154,13 +175,20 @@ test("步骤清单：只关挑刺与回应，协议照样冻结", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
test("步骤清单：只关消融，解释照样写，选格没有规则", () => {
|
||||
test("步骤清单：只关消融，解释也跳过，选格没有规则", () => {
|
||||
const list = listOf((f) => (f.ablation = { enabled: false }));
|
||||
assertReasons(list);
|
||||
assert.deepEqual(rows(list.study), rows(stepList(fullConfig()).study));
|
||||
assert.deepEqual(rows(list.round), [
|
||||
["openRound", E, null],
|
||||
["carryOverPool", E, null],
|
||||
["diagnosisProposal", S, null],
|
||||
["diagnosisDedupe", S, null],
|
||||
["diagnosisPrediction", S, null],
|
||||
["selectDiagnosis", S, null],
|
||||
["diagnosisImplementation", S, null],
|
||||
["diagnosisRun", S, null],
|
||||
["diagnosisSettle", S, null],
|
||||
["proposal", E, null],
|
||||
["dedupe", E, null],
|
||||
["firstPrediction", E, null],
|
||||
@@ -177,7 +205,7 @@ test("步骤清单：只关消融，解释照样写，选格没有规则", () =>
|
||||
["informationAccount", E, null],
|
||||
["adopt", E, null],
|
||||
["listCells", S, null],
|
||||
["explanation", E, null],
|
||||
["explanation", S, null],
|
||||
["selectCells", S, null],
|
||||
["ablationImplementation", S, null],
|
||||
["ablationVerification", S, null],
|
||||
@@ -205,6 +233,13 @@ test("步骤清单：朴素循环", () => {
|
||||
assert.deepEqual(rows(list.round), [
|
||||
["openRound", E, null],
|
||||
["carryOverPool", S, null], // 候选不留池
|
||||
["diagnosisProposal", S, null], // 不诊断
|
||||
["diagnosisDedupe", S, null], // 不诊断
|
||||
["diagnosisPrediction", S, null], // 不诊断
|
||||
["selectDiagnosis", S, null], // 不诊断
|
||||
["diagnosisImplementation", S, null], // 不诊断
|
||||
["diagnosisRun", S, null], // 不诊断
|
||||
["diagnosisSettle", S, null], // 不诊断
|
||||
["proposal", E, null],
|
||||
["dedupe", E, null], // 判重照常
|
||||
["firstPrediction", S, null], // 不预测
|
||||
@@ -264,3 +299,30 @@ test("步骤清单：换规则后清单里的规则跟着换", () => {
|
||||
assert.deepEqual(rule("selectBatch"), { name: "uniformRandom", version: 1 });
|
||||
assert.deepEqual(rule("selectCells"), { name: "randomSameCount", version: 1 });
|
||||
});
|
||||
|
||||
test("步骤清单：诊断开、消融关（现行配置）：诊断七步执行，消融七步（含解释）跳过", () => {
|
||||
const list = listOf((f) => {
|
||||
f.diagnosis = { enabled: true, perResearcher: 2, maxExperiments: 4 };
|
||||
f.ablation = { enabled: false };
|
||||
});
|
||||
assertReasons(list);
|
||||
const status = (id: string) => list.round.find((s) => s.id === id)!.status;
|
||||
for (const id of [
|
||||
"diagnosisProposal", "diagnosisDedupe", "diagnosisPrediction", "selectDiagnosis",
|
||||
"diagnosisImplementation", "diagnosisRun", "diagnosisSettle",
|
||||
]) {
|
||||
assert.equal(status(id), "execute", id);
|
||||
}
|
||||
for (const id of [
|
||||
"listCells", "explanation", "selectCells", "ablationImplementation", "ablationVerification", "ablationRun", "settle",
|
||||
]) {
|
||||
assert.equal(status(id), "skip", id);
|
||||
}
|
||||
// 诊断七步排在留池之后、提案之前（方法.md 4.3"诊断"一段）。
|
||||
const ids = list.round.map((s) => s.id);
|
||||
assert.deepEqual(ids.slice(ids.indexOf("carryOverPool"), ids.indexOf("proposal") + 1), [
|
||||
"carryOverPool", "diagnosisProposal", "diagnosisDedupe", "diagnosisPrediction", "selectDiagnosis",
|
||||
"diagnosisImplementation", "diagnosisRun", "diagnosisSettle", "proposal",
|
||||
]);
|
||||
assert.deepEqual(list.round.find((s) => s.id === "selectDiagnosis")!.rule, { name: "valueMinusCost", version: 1 });
|
||||
});
|
||||
|
||||
Vendored
+1937
File diff suppressed because it is too large
Load Diff
+318
@@ -0,0 +1,318 @@
|
||||
"""生成诊断（方法.md 6.12）的预期值 JSON（diagnosis.json）。
|
||||
|
||||
用法（在仓库根目录）：
|
||||
python3 tests/fixtures/rules/gen_diagnosis.py
|
||||
只用标准库；依赖同目录的 rules_ref.py 里的基础件（折线、有理数分布、Stieltjes 精确积分、比特、惊讶、分歧量）。
|
||||
重跑结果逐位相同。
|
||||
|
||||
独立性：诊断的成分模型、方向收益、信息价值、价值减成本、选实验、结算都在本文件里只照 docs/方法.md 6.12 和 6.7、6.9
|
||||
的文字写成，没有读过 src/rules/diagnosis.ts 和 src/rules/value.ts。
|
||||
|
||||
算法（有理数精确，浮点输入先精确转成有理数）：
|
||||
- 每个成分每个实验的分布：各条解释的折线加这个实验自己的开放预留（6.3"诊断"），换成收益 g = 改进量（x − b，越小越好时 b − x）；
|
||||
- E(g_d)：各成分期望按起始权重平均；
|
||||
- 全做（k = 实验总数）的 VOI：Σ_j w_j · E_j[max(0, max_d g_d)]（rules_ref.expect_max 的 floor0 模式）− max(0, max_d E g_d)；
|
||||
- 只做一个实验 d0 的 VOI（exactK1）：d0 的各成分在相邻折点之间密度是常数，所以后验权重在每一段里是常数，
|
||||
没做的实验的期望收益 c_e 在每一段里是常数，max(0, v, c_e…) 对 v 的积分有闭式；
|
||||
- 价值 = u·m′·VOI，R̂ = ⌈(余额 − 1) ÷ K⌉，成本 = 2·max(0, ĝ₀)·[(K+1)/(K+2) − K/(K+1)]；
|
||||
- 推出的方向：成分新权重 ∝ 起始权重 × 各有效实验在实测处的密度（所在段斜率，段左闭右开）。
|
||||
|
||||
生成的文件：diagnosis.json，格式：
|
||||
{"generator", "tolerance", "scenarios": [{"name", "section", "input", "expected", "note"}]}
|
||||
- 场景 input：{"b", "direction", "lambda", "experiments": ["d1", ...],
|
||||
"explanations": [{"id", "author", "ranges": {"d1": {"q1","q5","q9"}, ...}}]}，
|
||||
选实验场景另带 "balance"、"batchCap"、"maxExperiments"、"u"、"decay"，结算场景另带 "epsilon" 和 "outcomes"。
|
||||
- 分数一律是原值；收益、VOI 是收益刻度（改进量）。
|
||||
"""
|
||||
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import sys
|
||||
from fractions import Fraction as Fr
|
||||
from itertools import combinations
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import rules_ref as R # noqa: E402
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
F = float
|
||||
|
||||
|
||||
def tri(t):
|
||||
return {"q1": t[0], "q5": t[1], "q9": t[2]}
|
||||
|
||||
|
||||
def ids_key(s):
|
||||
return R.factor_key(s)
|
||||
|
||||
|
||||
class Model:
|
||||
"""诊断的成分模型（有理数）。explanations 按编号排好；成分 = 各解释 + 开放预留。"""
|
||||
|
||||
def __init__(self, b, direction, lam, experiments, explanations):
|
||||
self.b, self.direction, self.lam = fr(b), direction, fr(lam)
|
||||
self.exps = sorted(experiments, key=ids_key)
|
||||
self.expl = sorted(explanations, key=lambda e: ids_key(e["id"]))
|
||||
authors = [e["author"] for e in self.expl]
|
||||
self.w = R.explanation_weights(authors, lam) + [fr(lam)]
|
||||
# raw[d][j]：分数原值上的点列；gain[d][j]：收益（改进量）上的点列
|
||||
self.raw, self.gain = {}, {}
|
||||
for d in self.exps:
|
||||
pts = [R.polyline(*e["ranges"][d]) for e in self.expl]
|
||||
lo, hi = R.reserve_bounds(pts)
|
||||
pts.append(R.uniform_pts(lo, hi))
|
||||
self.raw[d] = pts
|
||||
self.gain[d] = [R.improvement_pts(p, self.b, direction) for p in pts]
|
||||
self.mu = {d: [R.mean(R.single(p)) for p in self.gain[d]] for d in self.exps}
|
||||
self.prior = {d: sum((wj * m for wj, m in zip(self.w, self.mu[d])), Fr(0)) for d in self.exps}
|
||||
|
||||
def g0(self):
|
||||
return max(Fr(0), max(self.prior.values()))
|
||||
|
||||
def voi_all(self):
|
||||
total = Fr(0)
|
||||
for j, wj in enumerate(self.w):
|
||||
if wj == 0:
|
||||
continue
|
||||
total += wj * R.expect_max([R.single(self.gain[d][j]) for d in self.exps], floor0=True)
|
||||
return total - self.g0()
|
||||
|
||||
def voi_k1(self, d0):
|
||||
"""只做 d0：每一段里后验权重和没做的实验的期望收益都是常数，对 max(0, v, c…) 精确积分。"""
|
||||
bps = sorted(set().union(*[set(x for x, _ in p) for p in self.gain[d0]]))
|
||||
total = Fr(0)
|
||||
for a, b in zip(bps, bps[1:]):
|
||||
dens = [R.pts_left(p, b) - R.pts_left(p, a) for p in self.gain[d0]]
|
||||
dens = [x / (b - a) for x in dens]
|
||||
marg = sum((wj * x for wj, x in zip(self.w, dens)), Fr(0))
|
||||
if marg == 0:
|
||||
continue
|
||||
c0 = Fr(0)
|
||||
for e in self.exps:
|
||||
if e == d0:
|
||||
continue
|
||||
num = sum((wj * x * m for wj, x, m in zip(self.w, dens, self.mu[e])), Fr(0))
|
||||
c0 = max(c0, num / marg)
|
||||
t = min(max(c0, a), b)
|
||||
total += marg * (c0 * (t - a) + (b * b - t * t) / 2)
|
||||
return total - self.g0()
|
||||
|
||||
def mixture(self, d):
|
||||
"""实验 d 的成分混合（分数原值）。"""
|
||||
return R.Dist(list(zip(self.w, self.raw[d])))
|
||||
|
||||
|
||||
def fr(x):
|
||||
return R.fr(x)
|
||||
|
||||
|
||||
def model_of(inp):
|
||||
expl = [{"id": e["id"], "author": e["author"], "ranges": {d: (r["q1"], r["q5"], r["q9"]) for d, r in e["ranges"].items()}} for e in inp["explanations"]]
|
||||
return Model(inp["b"], inp["direction"], inp["lambda"], inp["experiments"], expl)
|
||||
|
||||
|
||||
def explanation(eid, author, ranges):
|
||||
return {"id": eid, "author": author, "ranges": {d: tri(t) for d, t in ranges.items()}}
|
||||
|
||||
|
||||
def m_prime(balance, K, r):
|
||||
R_ = -(-(balance - 1) // K)
|
||||
return R_, sum((fr(r) ** j for j in range(R_)), Fr(0))
|
||||
|
||||
|
||||
def cost_of(g0, K):
|
||||
return 2 * g0 * (Fr(K + 1, K + 2) - Fr(K, K + 1))
|
||||
|
||||
|
||||
S = []
|
||||
|
||||
|
||||
def select_case(name, inp, balance, K, D, u, r, note):
|
||||
inp = dict(inp, balance=balance, batchCap=K, maxExperiments=D, u=u, decay=r)
|
||||
m = model_of(inp)
|
||||
rows = []
|
||||
if balance >= 2:
|
||||
size = min(len(m.exps), D)
|
||||
R_, mp = m_prime(balance, K, r)
|
||||
c = cost_of(m.g0(), K)
|
||||
for combo in combinations(m.exps, size):
|
||||
if size == len(m.exps):
|
||||
voi = m.voi_all()
|
||||
elif size == 1:
|
||||
voi = m.voi_k1(combo[0])
|
||||
else:
|
||||
voi = None # 网格近似，没有精确值，只检查结构
|
||||
if voi is None:
|
||||
rows.append({"experiments": list(combo), "voi": None})
|
||||
continue
|
||||
value = fr(u) * mp * voi
|
||||
rows.append({"experiments": list(combo), "voi": F(voi), "rHat": R_, "mPrime": F(mp), "value": F(value),
|
||||
"cost": F(c), "net": F(value - c)})
|
||||
expected = {
|
||||
"priorGains": {d: F(m.prior[d]) for d in m.exps},
|
||||
"g0": F(m.g0()),
|
||||
"weights": [F(x) for x in m.w],
|
||||
"combos": rows,
|
||||
}
|
||||
if all(r_["voi"] is not None for r_ in rows) and rows:
|
||||
eligible = [r_ for r_ in rows if r_["net"] > 1e-12]
|
||||
if eligible:
|
||||
top = max(r_["net"] for r_ in eligible)
|
||||
near = [r_ for r_ in eligible if r_["net"] >= top - 1e-12]
|
||||
best = min(near, key=lambda r_: [ids_key(x) for x in r_["experiments"]])
|
||||
expected["diagnose"] = True
|
||||
expected["chosen"] = best["experiments"]
|
||||
expected["chosenNet"] = best["net"]
|
||||
else:
|
||||
expected["diagnose"] = False
|
||||
expected["chosen"] = []
|
||||
expected["chosenNet"] = 0.0
|
||||
elif not rows:
|
||||
expected["diagnose"] = False
|
||||
expected["chosen"] = []
|
||||
expected["chosenNet"] = 0.0
|
||||
S.append({"name": name, "section": "6.12", "input": inp, "expected": expected, "note": note})
|
||||
|
||||
|
||||
def settle_case(name, inp, outcomes, note):
|
||||
inp = dict(inp, outcomes=outcomes)
|
||||
m = model_of(inp)
|
||||
eps = fr(inp["epsilon"])
|
||||
delta = F(eps / 100)
|
||||
b = F(m.b)
|
||||
scores = {d: (o["score"] if o["kind"] == "valid" else None) for d, o in outcomes.items()}
|
||||
rows = {}
|
||||
for d in m.exps:
|
||||
dist = m.mixture(d)
|
||||
row = {"outcome": outcomes[d]["kind"]}
|
||||
row["merged"] = {"q1": F(R.quantile(dist, Fr(1, 10))), "q5": F(R.quantile(dist, Fr(1, 2))), "q9": F(R.quantile(dist, Fr(9, 10)))}
|
||||
if scores[d] is not None:
|
||||
x = scores[d]
|
||||
row["gain"] = F(R.improve(x, m.b, m.direction))
|
||||
row["worldBits"] = R.world_bits(dist, b, delta)
|
||||
sp = R.surprise(dist, x, b, delta)
|
||||
row["surprise"] = sp
|
||||
persons = {}
|
||||
for e in m.expl:
|
||||
persons.setdefault(e["author"], []).append(R.polyline(*e["ranges"][d]))
|
||||
person_dists = [R.Dist([(Fr(1, len(ps)), p) for p in ps]) for _, ps in sorted(persons.items(), key=lambda kv: ids_key(kv[0]))]
|
||||
row["disagreement"] = R.disagreement(person_dists, b, delta)[0]
|
||||
rows[d] = row
|
||||
# 确认与否定：只有有效实验判，落在 [q1, q9]（含两端）确认。
|
||||
judgments = {}
|
||||
for e in m.expl:
|
||||
judgments[e["id"]] = {d: ("confirmed" if e["ranges"][d][0] <= scores[d] <= e["ranges"][d][2] else "denied")
|
||||
for d in m.exps if scores[d] is not None}
|
||||
# 诊断推出的方向
|
||||
new = []
|
||||
for j, wj in enumerate(m.w):
|
||||
v = wj
|
||||
for d in m.exps:
|
||||
if scores[d] is not None:
|
||||
v *= R.pts_density(m.raw[d][j], fr(scores[d]))
|
||||
new.append(v)
|
||||
tot = sum(new, Fr(0))
|
||||
kept = tot == 0
|
||||
w2 = m.w if kept else [v / tot for v in new]
|
||||
gains = {}
|
||||
for d in m.exps:
|
||||
if scores[d] is not None:
|
||||
gains[d] = {"gain": F(R.improve(scores[d], m.b, m.direction)), "measured": True}
|
||||
else:
|
||||
gains[d] = {"gain": F(sum((wj * mu for wj, mu in zip(w2, m.mu[d])), Fr(0))), "measured": False}
|
||||
best = m.exps[0]
|
||||
for d in m.exps[1:]:
|
||||
if gains[d]["gain"] > gains[best]["gain"]:
|
||||
best = d
|
||||
expected = {"experiments": rows, "judgments": judgments,
|
||||
"direction": {"weights": [F(x) for x in w2], "weightsKept": kept, "gains": gains,
|
||||
"experiment": best, "gain": gains[best]["gain"], "found": gains[best]["gain"] > 0}}
|
||||
S.append({"name": name, "section": "6.12 结算；6.9", "input": inp, "expected": expected, "note": note})
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ 场景
|
||||
|
||||
def base(explanations, experiments, b=0.0, direction="maximize", lam=0.1):
|
||||
return {"b": b, "direction": direction, "lambda": lam, "experiments": experiments, "explanations": explanations}
|
||||
|
||||
|
||||
# 1. 两条解释对同一个实验预测相反：A 说涨 0.2 左右，B 说跌 0.2 左右。
|
||||
OPP = [explanation("e1", "researcher-1", {"d1": (0.15, 0.2, 0.25)}),
|
||||
explanation("e2", "researcher-2", {"d1": (-0.25, -0.2, -0.15)})]
|
||||
select_case("opposite_lambda0", base(OPP, ["d1"], lam=0.0), 7, 2, 4, 0.5, 0.5,
|
||||
"λ = 0，两个成分的范围不相交：做了就知道是哪一条。手算：做了以后 E[max(0, g)] = 0.5 × 0.2 = 0.1，"
|
||||
"不做时 max(0, E g) = max(0, 0.5×0.2 + 0.5×(−0.2)) = 0，VOI = 0.1。")
|
||||
select_case("opposite_lambda01", base(OPP, ["d1"]), 7, 2, 4, 0.5, 0.5, "同上，λ = 0.1，开放预留铺在两端之外。")
|
||||
|
||||
# 2. 两条解释预测一样：λ = 0 时信息价值为 0；λ = 0.1 时只剩开放预留带来的一点。
|
||||
SAME = [explanation("e1", "researcher-1", {"d1": (0.1, 0.2, 0.3)}),
|
||||
explanation("e2", "researcher-2", {"d1": (0.1, 0.2, 0.3)})]
|
||||
select_case("same_lambda0", base(SAME, ["d1"], lam=0.0), 7, 2, 4, 0.5, 0.5,
|
||||
"λ = 0，两条解释一样，做不做都不改变看法：VOI = 0（手算：g 的取值范围 0 到 0.4 全在 0 以上，E[max(0, g)] = E g = 0.2）。净值 = −成本 < 0，不诊断。")
|
||||
select_case("same_lambda01", base(SAME, ["d1"]), 7, 2, 4, 0.5, 0.5, "λ = 0.1：解释们的范围都在 0 以上，只有开放预留（铺在 −0.2 到 0.6）能给出负的结果，所以 VOI 只剩预留带来的一小部分。")
|
||||
|
||||
# 3. 三个实验、三条解释，全做（3 ≤ D）。
|
||||
MULTI = [explanation("e1", "researcher-1", {"d1": (0.02, 0.05, 0.08), "d2": (-0.05, 0.0, 0.05), "d3": (0.0, 0.1, 0.2)}),
|
||||
explanation("e2", "researcher-2", {"d1": (0.0, 0.03, 0.06), "d2": (0.1, 0.2, 0.3), "d3": (-0.1, -0.05, 0.0)}),
|
||||
explanation("e3", "researcher-3", {"d1": (-0.02, 0.04, 0.1), "d2": (-0.2, -0.1, 0.0), "d3": (0.05, 0.1, 0.15)})]
|
||||
select_case("multi_all", base(MULTI, ["d1", "d2", "d3"]), 9, 2, 4, 0.5, 0.5,
|
||||
"三个实验不超过 D = 4，只有全做一个组合。R̂ = ⌈8 ÷ 2⌉ = 4，m′ = 1.875，成本 = 2ĝ₀ × (3/4 − 2/3)。")
|
||||
|
||||
# 4. 实验数超过 D：只比较装满的组合。D = 1 时是三个单实验组合（有精确值）；D = 2 时只有三个两实验组合（只查结构）。
|
||||
select_case("over_cap_d1", base(MULTI, ["d1", "d2", "d3"]), 9, 2, 1, 0.5, 0.5,
|
||||
"三个实验超过 D = 1：只有三个单实验组合，没有空组合；精确值用逐段闭式积分。")
|
||||
select_case("over_cap_d2", base(MULTI, ["d1", "d2", "d3"]), 9, 2, 2, 0.5, 0.5,
|
||||
"三个实验超过 D = 2：只有三个两实验组合，没有单实验组合；只检查组合列表（网格近似没有精确值）。")
|
||||
|
||||
# 5. 余额不到 2 次不诊断。
|
||||
select_case("balance_one", base(MULTI, ["d1", "d2", "d3"]), 1, 2, 4, 0.5, 0.5, "余额 1：没有可选组合，不诊断。")
|
||||
|
||||
# 6. 平票按实验编号：d2 和 d10 的预测完全一样，D = 1 时两个单实验组合净值相同，取 d2（自然序，d2 排在 d10 前面）。
|
||||
TIE = [explanation("e1", "researcher-1", {"d2": (-0.2, 0.0, 0.2), "d10": (-0.2, 0.0, 0.2)}),
|
||||
explanation("e2", "researcher-2", {"d2": (0.1, 0.3, 0.5), "d10": (0.1, 0.3, 0.5)})]
|
||||
select_case("tie_by_id", base(TIE, ["d10", "d2"]), 7, 2, 1, 0.5, 0.5, "两个实验的预测完全一样，净值相同（差不到 1e-12），取实验编号小的 d2。")
|
||||
|
||||
# 7. 结算。
|
||||
def shifted(expls, b):
|
||||
"""把以基线为 0 的改进量范围搬成分数原值（基线 b）。"""
|
||||
return [dict(e, ranges={d: {k: v + b for k, v in r.items()} for d, r in e["ranges"].items()}) for e in expls]
|
||||
|
||||
|
||||
SETTLE = shifted(MULTI[:2] + [explanation("e3", "researcher-2", {"d1": (-0.02, 0.04, 0.1), "d2": (-0.2, -0.1, 0.0), "d3": (0.05, 0.1, 0.15)})], 1.0)
|
||||
SINP = dict(base(SETTLE, ["d1", "d2", "d3"], b=1.0), epsilon=0.01)
|
||||
settle_case("settle_mixed", SINP,
|
||||
{"d1": {"kind": "valid", "score": 1.045}, "d2": {"kind": "insufficient"}, "d3": {"kind": "untested"}},
|
||||
"d1 有效，d2 证据不足（不判、不记账），d3 没做。研究者 2 交了两条解释，各占他那份的一半。")
|
||||
settle_case("settle_two_valid", SINP,
|
||||
{"d1": {"kind": "valid", "score": 1.07}, "d2": {"kind": "valid", "score": 1.25}, "d3": {"kind": "untested"}},
|
||||
"两个有效实验：权重按两处密度连乘更新；d2 的实测 +0.25 落在 e2 的范围里、其余两条之外。")
|
||||
settle_case("settle_no_direction", dict(SINP),
|
||||
{"d1": {"kind": "valid", "score": 0.97}, "d2": {"kind": "valid", "score": 0.8}, "d3": {"kind": "valid", "score": 0.9}},
|
||||
"全部实验的收益都不为正（−0.03、−0.2、−0.1）：最大收益 −0.03 ≤ 0，记没有指出能涨分的方向。")
|
||||
settle_case("settle_all_insufficient", SINP,
|
||||
{"d1": {"kind": "insufficient"}, "d2": {"kind": "insufficient"}, "d3": {"kind": "insufficient"}},
|
||||
"没有一个有效实验：不判、不记账，权重保持起始权重，方向按起始权重估的期望收益。")
|
||||
settle_case("settle_outside_all", SINP,
|
||||
{"d1": {"kind": "valid", "score": 3.0}, "d2": {"kind": "untested"}, "d3": {"kind": "untested"}},
|
||||
"d1 实测 +2.0 落在所有成分（含开放预留）的范围之外，权重无从更新：保持起始权重（weightsKept）。")
|
||||
|
||||
MIN_EXPL = [explanation("e1", "researcher-1", {"d1": (0.95, 0.9, 0.85)[::-1], "d2": (0.9, 1.0, 1.1)}),
|
||||
explanation("e2", "researcher-2", {"d1": (0.8, 0.85, 0.9), "d2": (0.95, 1.05, 1.15)})]
|
||||
MIN_INP = dict(base(MIN_EXPL, ["d1", "d2"], b=1.0, direction="minimize"), epsilon=0.01)
|
||||
select_case("minimize_all", dict(base(MIN_EXPL, ["d1", "d2"], b=1.0, direction="minimize")), 7, 2, 4, 0.5, 0.5,
|
||||
"越小越好：收益 = b − x。d1 两条解释都预测降到 0.9 左右（收益约 +0.1），d2 预测上升。")
|
||||
settle_case("settle_minimize", MIN_INP, {"d1": {"kind": "valid", "score": 0.88}, "d2": {"kind": "valid", "score": 1.05}},
|
||||
"越小越好：d1 实测 0.88，收益 = 1.0 − 0.88 = 0.12；d2 实测 1.05，收益 −0.05。")
|
||||
|
||||
doc = {"generator": "gen_diagnosis.py",
|
||||
"tolerance": {
|
||||
"exact": "有理数精确算出的量（成分权重、事前期望收益、ĝ₀、全做的 VOI、R̂、m′、成本、价值、净值、收益、更新后的权重）：绝对误差 1e-12",
|
||||
"exactK1": "只做一个实验的 VOI 是逐段闭式积分，实现用部分格网格近似：|差| ≤ 1e-9 + 2e-4·|精确值|",
|
||||
"bits": "世界比特、惊讶、分歧量是浮点：绝对误差 1e-9；合并范围是分位点：绝对误差 1e-9",
|
||||
"discrete": "是否诊断、选中的实验、确认与否定、有没有指出方向、实验的结局：必须完全相同"},
|
||||
"scenarios": S}
|
||||
with open(os.path.join(HERE, "diagnosis.json"), "w", encoding="utf-8") as f:
|
||||
json.dump(doc, f, ensure_ascii=False, indent=1)
|
||||
f.write("\n")
|
||||
print(len(S), "个场景")
|
||||
@@ -0,0 +1,230 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { test, type TestContext } from "node:test";
|
||||
import { stepList } from "../src/config/index.ts";
|
||||
import { fullConfig as checkedFull } from "../src/flow/frozen.ts";
|
||||
import { judgeDiagnosisRun } from "../src/flow/formal-run.ts";
|
||||
import { containerName, freshDir, runPaths, workspaceRepo } from "../src/flow/paths.ts";
|
||||
import { recover } from "../src/flow/recover.ts";
|
||||
import { diagnosisRun, diagnosisSettle } from "../src/flow/steps/diagnosis.ts";
|
||||
import type { Study } from "../src/flow/study.ts";
|
||||
import { Ledger } from "../src/ledger/index.ts";
|
||||
import { Board } from "../src/materials/index.ts";
|
||||
import { runBalance, settleDiagnosis, type SettleDiagnosisInput } from "../src/rules/index.ts";
|
||||
import { Runner } from "../src/runner/index.ts";
|
||||
import { diagnosisRun as diagnosisDefinition, diagnosisUnversionedNames, loadTask, prepareDiagnosisRun, type DiagnosisJudged } from "../src/tasks/index.ts";
|
||||
import { checkoutVersion, createRepository, saveVersion } from "../src/workspace/index.ts";
|
||||
import { fullConfig } from "./config-helpers.ts";
|
||||
import { judged, r, Writer } from "./materials-helpers.ts";
|
||||
|
||||
// 诊断运行在 flow 里的几处，用真实 Docker 测，不花模型的钱（任务书 18 第 1.6 节）：
|
||||
// 选中的实验拿掉一个后跑的只是剩下的实验、逐个实验判分和结算；诊断运行恢复只核对、不重做、只扣一次。
|
||||
// 镜像一律 inforsi-test-flow-dx 开头的固定名字，只动自己前缀的。判分的预期值是手算的相切布局：26 个半径 1/64 的圆排成一行，
|
||||
// 圆心间距 1/32，半径和 26/64 = 0.40625（同 tests/tasks-diagnosis.test.ts，都是 2 的幂有理数，没有舍入）。
|
||||
|
||||
const TASK_DIR = join(import.meta.dirname, "..", "tasks", "circle_packing");
|
||||
const BASELINE_SUM = 26 / 64;
|
||||
|
||||
function docker(args: string[]): { status: number | null; stderr: string } {
|
||||
return spawnSync("docker", args, { encoding: "utf8" });
|
||||
}
|
||||
|
||||
function removeImage(tag: string): void {
|
||||
const removed = docker(["rmi", tag]);
|
||||
if (removed.status !== 0 && !removed.stderr.includes("No such image")) throw new Error(removed.stderr);
|
||||
}
|
||||
|
||||
function tempDir(t: TestContext): string {
|
||||
const dir = mkdtempSync(join(tmpdir(), "inforsi-flow-dx-"));
|
||||
t.after(() => rmSync(dir, { recursive: true, force: true }));
|
||||
return dir;
|
||||
}
|
||||
|
||||
/** 诊断程序：d1 原样交回基线的原始答案；别的实验编号被调用就崩（用来证明拿掉的实验没被调用）。 */
|
||||
const PROGRAM = `import json
|
||||
|
||||
|
||||
def run_experiment(experiment_id, baseline_dir):
|
||||
if experiment_id == "d1":
|
||||
with open(baseline_dir + "/solution.json") as handle:
|
||||
solution = json.load(handle)
|
||||
return solution["centers"], solution["radii"], solution["reported_sum"]
|
||||
raise RuntimeError("不该被调用的实验 " + experiment_id)
|
||||
`;
|
||||
|
||||
/** 真实任务镜像、账本、裸仓库；第 0 轮基线已跑完（运行目录里放着相切布局的原始答案，半径和 0.40625），诊断程序已存成版本。 */
|
||||
async function setup(t: TestContext, suffix: string, maxRepairs: number): Promise<{ s: Study; w: Writer; version: string }> {
|
||||
const dir = tempDir(t);
|
||||
const id = `test-flow-dx-${suffix}`;
|
||||
removeImage(`inforsi-${id}`);
|
||||
t.after(() => removeImage(`inforsi-${id}`));
|
||||
const task = loadTask(TASK_DIR);
|
||||
const image = await new Runner(null).buildImage(task.dir, `inforsi-${id}`);
|
||||
const ledger = Ledger.open(join(dir, "ledger.sqlite"));
|
||||
t.after(() => ledger.close());
|
||||
const config = { ...fullConfig(), studyId: id, maxRepairs };
|
||||
if (config.flow.preset !== "full") throw new Error("测试配置应是完整流程");
|
||||
config.flow = { ...config.flow, diagnosis: { enabled: true, perResearcher: 2, maxExperiments: 4 }, ablation: { enabled: false } };
|
||||
const s = { dir, task, runner: new Runner(null), ledger, stop: new AbortController().signal,
|
||||
frozen: { config: checkedFull(config), steps: stepList(config), facts: task.facts, prompts: null, image, seed: 1 } } as unknown as Study;
|
||||
const w = new Writer(ledger);
|
||||
ledger.append("code", { object: "base", version: "v-base", locations: null });
|
||||
w.run("base-1", "baseline", 0, null, judged(BASELINE_SUM, "main"));
|
||||
const baseRun = runPaths(dir, "base-1").run;
|
||||
mkdirSync(baseRun, { recursive: true });
|
||||
writeFileSync(join(baseRun, "solution.json"), JSON.stringify({
|
||||
centers: Array.from({ length: 26 }, (_, i) => [7 / 64 + i / 32, 0.5]),
|
||||
radii: Array(26).fill(1 / 64),
|
||||
reported_sum: 99,
|
||||
}));
|
||||
createRepository(workspaceRepo(dir));
|
||||
const program = join(dir, "program");
|
||||
mkdirSync(program);
|
||||
writeFileSync(join(program, "initial.py"), PROGRAM);
|
||||
const saved = saveVersion(workspaceRepo(dir), program, diagnosisUnversionedNames(task));
|
||||
if (!saved.saved) throw new Error(`诊断程序存不成版本：${saved.reason}`);
|
||||
return { s, w, version: saved.version };
|
||||
}
|
||||
|
||||
const RANGES = {
|
||||
a1: [r(0.5, 0.6, 0.7), r(0.3, 0.4, 0.45)],
|
||||
a2: [r(0.2, 0.3, 0.4), r(0.3, 0.4, 0.45)],
|
||||
} as const;
|
||||
|
||||
/** 本轮诊断写到"选中 d1、d2，核验只有 d1 通过"为止：诊断解释与实验、判重合并、诊断预测、选实验、诊断程序、核验。 */
|
||||
function selectedAndVerified({ w, version }: { w: Writer; version: string }): void {
|
||||
const experiment = (name: string) => ({ name, spot: `${name}的位置`, body: `${name}的做法`, metrics: [] });
|
||||
w.open("r1-diagnosisProposal", "diagnosisProposal", 1, ["a1", "a2"]);
|
||||
for (const [author, name] of [["a1", "实验甲"], ["a2", "实验乙"]] as const) {
|
||||
w.sub("r1-diagnosisProposal", author, "diagnosisProposal", null, null, { explanations: [{ body: `${author}的解释` }], experiments: [experiment(name)] });
|
||||
}
|
||||
w.close("r1-diagnosisProposal");
|
||||
w.decision("diagnosisDedup", 1, null, {
|
||||
experiments: [{ experiment: "d1", sources: [{ author: "a1", index: 0 }] }, { experiment: "d2", sources: [{ author: "a2", index: 0 }] }],
|
||||
});
|
||||
w.open("r1-diagnosisPrediction", "diagnosisPrediction", 1, ["a1", "a2"]);
|
||||
for (const author of ["a1", "a2"] as const) {
|
||||
w.sub("r1-diagnosisPrediction", author, "diagnosisPrediction", null, null, {
|
||||
predictions: ["d1", "d2"].map((d, i) => ({ explanation: `r1-${author}-dx1`, experiment: d, range: RANGES[author][i]! })),
|
||||
body: "理由",
|
||||
});
|
||||
}
|
||||
w.close("r1-diagnosisPrediction");
|
||||
w.decision("selectDiagnosis", 1, null, { diagnose: true, chosen: ["d1", "d2"] });
|
||||
w.l.append("code", { object: "r1-diagnosis", version, locations: [] });
|
||||
w.open("r1-diagnosisVerification-1", "diagnosisVerification", 1, ["verifier-r1-diagnosis"]);
|
||||
w.sub("r1-diagnosisVerification-1", "verifier-r1-diagnosis", "diagnosisVerification", "r1-diagnosis", null, {
|
||||
experiments: [
|
||||
{ experiment: "d1", followed: true, nothingElse: true, where: "initial.py:4" },
|
||||
{ experiment: "d2", followed: false, nothingElse: true, where: "没找到" },
|
||||
],
|
||||
body: "无",
|
||||
});
|
||||
w.close("r1-diagnosisVerification-1");
|
||||
}
|
||||
|
||||
test("诊断运行：核验没通过又没有修复次数的实验拿掉，只跑剩下的实验；逐个实验判分，结算里拿掉的记证据不足", { timeout: 180_000 }, async (t) => {
|
||||
const fixture = await setup(t, "settle", 0);
|
||||
const { s } = fixture;
|
||||
selectedAndVerified(fixture);
|
||||
t.after(() => { docker(["rm", "-f", containerName(s.frozen.config.studyId, "r1-diagnosis-1")]); });
|
||||
assert.equal(await diagnosisRun(s, 1, "r1-diagnosis-1"), "done");
|
||||
const records = s.ledger.records();
|
||||
const planned = records.find((x) => x.kind === "runPlanned" && x.content.purpose === "diagnosis")!;
|
||||
if (planned.kind !== "runPlanned") throw new Error("缺登记");
|
||||
assert.deepEqual({ round: planned.content.round, object: planned.content.object }, { round: 1, object: "r1-diagnosis" });
|
||||
// 只调 d1，总时限 120 秒由它一个实验独享。
|
||||
const { command } = planned.content.definition;
|
||||
assert.deepEqual(command.slice(command.indexOf("--experiments")), ["--experiments", "d1"]);
|
||||
assert.equal(command[command.indexOf("--seconds-each") + 1], "120");
|
||||
assert.equal(records.filter((x) => x.kind === "runStarted" && x.content.run === "r1-diagnosis-1").length, 1);
|
||||
const result = records.findLast((x) => x.kind === "runResult")!;
|
||||
if (result.kind !== "runResult") throw new Error("缺结果");
|
||||
assert.equal(result.content.process.ending, "exited");
|
||||
assert.equal(result.content.judged, null);
|
||||
assert.deepEqual(runBalance(13, new Board(records).runStates()), { charged: 2, reserved: 0, balance: 11 });
|
||||
|
||||
await diagnosisSettle(s, 1);
|
||||
const settled = s.ledger.records().find((x) => x.kind === "diagnosisSettlement")!;
|
||||
if (settled.kind !== "diagnosisSettlement") throw new Error("缺结算");
|
||||
const input: SettleDiagnosisInput = {
|
||||
baseline: BASELINE_SUM,
|
||||
direction: "maximize",
|
||||
epsilon: s.frozen.facts.epsilon,
|
||||
lambda: 0.1,
|
||||
experiments: [
|
||||
{ experiment: "d1", outcome: { kind: "valid", score: BASELINE_SUM } },
|
||||
{ experiment: "d2", outcome: { kind: "insufficient" } },
|
||||
],
|
||||
explanations: (["a1", "a2"] as const).map((author) => ({
|
||||
id: `r1-${author}-dx1`,
|
||||
author,
|
||||
experiments: ["d1", "d2"].map((experiment, i) => ({ experiment, range: RANGES[author][i]! })),
|
||||
})),
|
||||
};
|
||||
const { judged: j, dropped, run, ...rest } = settled.content.result as unknown as { judged: DiagnosisJudged; dropped: Array<{ experiment: string; reason: string }>; run: string };
|
||||
assert.equal(run, "r1-diagnosis-1");
|
||||
assert.deepEqual(rest, { input, output: settleDiagnosis(input) });
|
||||
// 判分只有 d1 一项：拿掉的 d2 没跑、没判。
|
||||
assert.equal(j.experiments.length, 1);
|
||||
const d1 = j.experiments[0]!;
|
||||
assert.deepEqual([d1.id, d1.ending, d1.verdict, d1.score], ["d1", "finished", "valid", { status: "measured", value: BASELINE_SUM }]);
|
||||
assert.deepEqual(dropped.map((d) => d.experiment), ["d2"]);
|
||||
assert.match(dropped[0]!.reason, /第 1 次核验仍不通过，修复次数上限 0 已用完：没照写法做；没找到/);
|
||||
// 结算写进去以后诊断运行公开；每轮最多一条结算。
|
||||
assert.ok(new Board(s.ledger.records()).runs().filter((x) => x.plan.purpose === "diagnosis").every((x) => new Board(s.ledger.records()).runPublic(x)));
|
||||
assert.throws(() => s.ledger.append("diagnosisSettlement", { round: 1, result: null }));
|
||||
});
|
||||
|
||||
/** 手工把一次诊断运行做到"登记了"：和 diagnosisRun 同样的目录准备，但不启动（启动由各个测试自己决定）。 */
|
||||
function planned(s: Study, version: string): { run: string; name: string; dir: string } {
|
||||
const run = "r1-diagnosis-1";
|
||||
const paths = runPaths(s.dir, run);
|
||||
const definition = diagnosisDefinition(s.task, s.frozen.image, String(s.frozen.seed), ["d1"]);
|
||||
s.ledger.append("runPlanned", { run, purpose: "diagnosis", round: 1, object: "r1-diagnosis", definition });
|
||||
freshDir(paths.base);
|
||||
checkoutVersion(workspaceRepo(s.dir), version, freshDir(paths.code));
|
||||
prepareDiagnosisRun(s.task, paths.code, runPaths(s.dir, "base-1").run, freshDir(paths.run), [{ id: "d1", metrics: [] }]);
|
||||
return { run, name: containerName(s.frozen.config.studyId, run), dir: paths.run };
|
||||
}
|
||||
|
||||
test("诊断运行恢复：容器已启动、账本没写确认启动和结果，接上原容器不重跑，只扣一次，运行记录不带判分，判分从运行目录重做", { timeout: 180_000 }, async (t) => {
|
||||
const { s, version } = await setup(t, "recover", 0);
|
||||
const { run, name, dir } = planned(s, version);
|
||||
t.after(() => { docker(["rm", "-f", name]); });
|
||||
assert.deepEqual(await s.runner.start(name, diagnosisDefinition(s.task, s.frozen.image, String(s.frozen.seed), ["d1"]), dir), { started: true });
|
||||
// 中断点：容器已启动，账本只有登记。恢复不调用 diagnosisRun，而是找到原容器等它。
|
||||
assert.equal(await recover(s), null);
|
||||
const records = s.ledger.records();
|
||||
assert.deepEqual(["runPlanned", "runStarted", "runResult"].map((kind) => records.filter((x) => x.kind === kind && (x.content as { run?: string }).run === run).length), [1, 1, 1]);
|
||||
const result = records.findLast((x) => x.kind === "runResult")!;
|
||||
if (result.kind !== "runResult") throw new Error("缺结果");
|
||||
assert.equal(result.content.run, run);
|
||||
assert.equal(result.content.process.ending, "exited");
|
||||
assert.equal(result.content.judged, null);
|
||||
assert.deepEqual(runBalance(13, new Board(records).runStates()), { charged: 2, reserved: 0, balance: 11 });
|
||||
assert.equal(await s.runner.status(name), "absent");
|
||||
assert.equal(await recover(s), null);
|
||||
assert.equal(s.ledger.records().length, records.length);
|
||||
// 恢复后的判分：从运行目录重判，逐个实验单独判分。
|
||||
const judgedRun = await judgeDiagnosisRun(s, run, result.content.process, [{ id: "d1", metrics: [] }]);
|
||||
assert.equal(judgedRun.experiments.length, 1);
|
||||
const d1 = judgedRun.experiments[0]!;
|
||||
assert.deepEqual([d1.id, d1.ending, d1.verdict, d1.score], ["d1", "finished", "valid", { status: "measured", value: BASELINE_SUM }]);
|
||||
});
|
||||
|
||||
test("诊断运行恢复：登记了、容器不存在，确认没启动：写结果没启动、退还名额，不补启动", { timeout: 120_000 }, async (t) => {
|
||||
const { s, version } = await setup(t, "notstarted", 0);
|
||||
planned(s, version);
|
||||
assert.equal(await recover(s), null);
|
||||
const records = s.ledger.records();
|
||||
const result = records.findLast((x) => x.kind === "runResult")!;
|
||||
if (result.kind !== "runResult") throw new Error("缺结果");
|
||||
assert.equal(result.content.process.ending, "notStarted");
|
||||
assert.equal(result.content.judged, null);
|
||||
assert.equal(records.filter((x) => x.kind === "runStarted" && x.content.run === "r1-diagnosis-1").length, 0);
|
||||
assert.deepEqual(runBalance(13, new Board(records).runStates()), { charged: 1, reserved: 0, balance: 12 });
|
||||
});
|
||||
@@ -0,0 +1,185 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { appendFileSync, cpSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { test, type TestContext } from "node:test";
|
||||
import { stepList } from "../src/config/index.ts";
|
||||
import { fullConfig as checkedFull } from "../src/flow/frozen.ts";
|
||||
import { runPaths, workspaceRepo } from "../src/flow/paths.ts";
|
||||
import { groupSameExperiment, selectDiagnosisExperiments } from "../src/flow/steps/diagnosis.ts";
|
||||
import { prepareSubmission, taskArtifact, taskCode } from "../src/flow/task-code.ts";
|
||||
import type { Study } from "../src/flow/study.ts";
|
||||
import { Ledger } from "../src/ledger/index.ts";
|
||||
import { runDiagnosisRule, type DiagnosisRuleInput } from "../src/rules/index.ts";
|
||||
import type { ForwardCode } from "../src/submissions/index.ts";
|
||||
import { checkTaskFacts, diagnosisUnversionedNames, loadTask, placeBaselineArtifact } from "../src/tasks/index.ts";
|
||||
import type { JevJudgment } from "../src/vocab/index.ts";
|
||||
import { compareVersions, createRepository, saveVersion } from "../src/workspace/index.ts";
|
||||
import { fullConfig } from "./config-helpers.ts";
|
||||
import { tempDir } from "./ledger-helpers.ts";
|
||||
import { judged, r, Writer } from "./materials-helpers.ts";
|
||||
|
||||
// 诊断里程序自己做的两步（任务书 18 第 1.6 节）：判重合并的分组、选诊断实验；加上诊断实现、修复、核验的取材和存版本接线。判重合并的 Jev 问法由 jev 的测试管；
|
||||
// 诊断运行和结算要真实 Docker，在 flow-diagnosis-docker.test.ts。预期值照条文手拼，输出用 rules 的函数当对照
|
||||
// （同选格测试的写法，rules 自己有独立预期值的测试）。
|
||||
|
||||
const facts = checkTaskFacts(JSON.parse(readFileSync(join(import.meta.dirname, "..", "tasks", "circle_packing", "task.json"), "utf8")));
|
||||
|
||||
const yes = (probability: number): JevJudgment => ({ question: "diagnosisDedup", probability, threshold: 0.3, yes: probability > 0.3 }) as unknown as JevJudgment;
|
||||
const item = (author: string, index: number) => ({ author, index, label: `${author}.e${index + 1}` });
|
||||
|
||||
test("诊断实验判重合并：判为相同的连成一组，编号 d1… 按组里排最前的实验排，组内按作者、第几个排", () => {
|
||||
const items = [item("researcher-1", 0), item("researcher-1", 1), item("researcher-2", 0), item("researcher-3", 0)];
|
||||
const pairs = [
|
||||
{ a: "researcher-1.e1", b: "researcher-2.e1", judgment: yes(0.9) },
|
||||
{ a: "researcher-1.e2", b: "researcher-3.e1", judgment: yes(0.1) },
|
||||
{ a: "researcher-2.e1", b: "researcher-3.e1", judgment: yes(0.8) },
|
||||
];
|
||||
// researcher-1.e1、researcher-2.e1、researcher-3.e1 连成一组（A=B，B=C）；researcher-1.e2 自成一个。
|
||||
assert.deepEqual(groupSameExperiment(items, pairs), {
|
||||
experiments: [
|
||||
{ experiment: "d1", sources: [{ author: "researcher-1", index: 0 }, { author: "researcher-2", index: 0 }, { author: "researcher-3", index: 0 }] },
|
||||
{ experiment: "d2", sources: [{ author: "researcher-1", index: 1 }] },
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
test("诊断实验判重合并：没有一对判相同时，每个实验各成一个", () => {
|
||||
const items = [item("researcher-1", 0), item("researcher-2", 0)];
|
||||
const pairs = [{ a: "researcher-1.e1", b: "researcher-2.e1", judgment: yes(0.1) }];
|
||||
assert.deepEqual(groupSameExperiment(items, pairs).experiments.map((e) => e.experiment), ["d1", "d2"]);
|
||||
});
|
||||
|
||||
/** 基线 0.96；a1、a2 各一条诊断解释，对合并后的 d1、d2 各给三个数；余额：13 次里基线用了 1 次。 */
|
||||
function study(t: TestContext, ranges: Record<string, ReturnType<typeof r>[]>): { s: Study; input: DiagnosisRuleInput } {
|
||||
const ledger = Ledger.open(join(tempDir(t), "ledger.db"));
|
||||
t.after(() => ledger.close());
|
||||
const w = new Writer(ledger);
|
||||
ledger.append("code", { object: "base", version: "v-base", locations: null });
|
||||
w.run("base-1", "baseline", 0, null, judged(0.96, "main"));
|
||||
w.open("r1-diagnosisProposal", "diagnosisProposal", 1, ["a1", "a2"]);
|
||||
const experiment = (name: string) => ({ name, spot: `${name}的位置`, body: `${name}的做法`, metrics: [] });
|
||||
for (const [author, name] of [["a1", "实验甲"], ["a2", "实验乙"]] as const) {
|
||||
w.sub("r1-diagnosisProposal", author, "diagnosisProposal", null, null, { explanations: [{ body: `${author}的解释` }], experiments: [experiment(name)] });
|
||||
}
|
||||
w.close("r1-diagnosisProposal");
|
||||
w.decision("diagnosisDedup", 1, null, {
|
||||
experiments: [{ experiment: "d1", sources: [{ author: "a1", index: 0 }] }, { experiment: "d2", sources: [{ author: "a2", index: 0 }] }],
|
||||
});
|
||||
w.open("r1-diagnosisPrediction", "diagnosisPrediction", 1, ["a1", "a2"]);
|
||||
for (const author of ["a1", "a2"]) {
|
||||
const mine = ranges[author]!;
|
||||
w.sub("r1-diagnosisPrediction", author, "diagnosisPrediction", null, null, {
|
||||
predictions: ["d1", "d2"].map((d, i) => ({ explanation: `r1-${author}-dx1`, experiment: d, range: mine[i]! })),
|
||||
body: "理由",
|
||||
});
|
||||
}
|
||||
w.close("r1-diagnosisPrediction");
|
||||
const config = fullConfig();
|
||||
if (config.flow.preset !== "full") throw new Error("测试配置应是完整流程");
|
||||
config.flow = { ...config.flow, diagnosis: { enabled: true, perResearcher: 2, maxExperiments: 4 }, ablation: { enabled: false } };
|
||||
const frozen = { config: checkedFull(config), steps: stepList(config), facts, prompts: null, image: "", seed: 1 };
|
||||
const input: DiagnosisRuleInput = {
|
||||
baseline: 0.96,
|
||||
direction: "maximize",
|
||||
lambda: 0.1,
|
||||
experiments: ["d1", "d2"],
|
||||
explanations: ["a1", "a2"].map((author) => ({
|
||||
id: `r1-${author}-dx1`,
|
||||
author,
|
||||
experiments: ["d1", "d2"].map((experiment, i) => ({ experiment, range: ranges[author]![i]! })),
|
||||
})),
|
||||
balance: 12,
|
||||
batchCap: 2,
|
||||
maxExperiments: 4,
|
||||
u: 0.5,
|
||||
decay: 0.5,
|
||||
};
|
||||
return { s: { ledger, frozen } as unknown as Study, input };
|
||||
}
|
||||
|
||||
test("选诊断实验：输入照 DiagnosisRuleInput 拼（基线、每条解释对每个实验的三个数、余额从本轮之前的运行算），规则取步骤清单，种子 null", (t) => {
|
||||
// 两人对 d1 预测相反（一个说涨到 1.3，一个说跌到 0.9），对 d2 都说涨不了多少。
|
||||
const { s, input } = study(t, {
|
||||
a1: [r(1.2, 1.3, 1.4), r(0.95, 0.97, 0.99)],
|
||||
a2: [r(0.8, 0.9, 1.0), r(0.95, 0.97, 0.99)],
|
||||
});
|
||||
selectDiagnosisExperiments(s, 1);
|
||||
const record = s.ledger.records().find((x) => x.kind === "decision" && x.content.decision === "selectDiagnosis")!;
|
||||
const { rule: used, ...output } = runDiagnosisRule("valueMinusCost", input);
|
||||
assert.deepEqual(record.content, { decision: "selectDiagnosis", round: 1, rule: "valueMinusCost", ruleVersion: used.version, params: output.params, seed: null, input, output });
|
||||
// 分歧大的实验信息价值为正：本轮做诊断，d1 在选中的组合里。
|
||||
assert.equal(output.diagnose, true);
|
||||
assert.ok(output.chosen.includes("d1"));
|
||||
});
|
||||
|
||||
/** 诊断实现的代码接线：真实裸仓库里存了基线；工作目录是基线副本加一处改动，再放进只读的原始产物。 */
|
||||
function codeStudy(t: TestContext): { s: Study; work: string; baseline: string } {
|
||||
const dir = tempDir(t);
|
||||
const ledger = Ledger.open(join(dir, "ledger.db"));
|
||||
t.after(() => ledger.close());
|
||||
const task = loadTask(join(import.meta.dirname, "..", "tasks", "circle_packing"));
|
||||
const repo = workspaceRepo(dir);
|
||||
createRepository(repo);
|
||||
const names = diagnosisUnversionedNames(task);
|
||||
const saved = saveVersion(repo, task.baselineDir, names);
|
||||
if (!saved.saved) throw new Error("基线存不成版本");
|
||||
const w = new Writer(ledger);
|
||||
ledger.append("code", { object: "base", version: saved.version, locations: null });
|
||||
w.run("base-1", "baseline", 0, null, judged(0.96, "main"));
|
||||
const baseRun = runPaths(dir, "base-1").run;
|
||||
mkdirSync(baseRun, { recursive: true });
|
||||
writeFileSync(join(baseRun, "solution.json"), "{}");
|
||||
w.open("r1-diagnosisProposal", "diagnosisProposal", 1, ["a1"]);
|
||||
const metric = { id: "note", meaning: "记录", unit: "无", direction: "maximize" as const, howMeasured: "数一数", missing: "recordMissing" as const };
|
||||
w.sub("r1-diagnosisProposal", "a1", "diagnosisProposal", null, null, { explanations: [{ body: "解释" }], experiments: [{ name: "实验甲", spot: "第一步", body: "换个做法", metrics: [metric] }] });
|
||||
w.close("r1-diagnosisProposal");
|
||||
w.decision("diagnosisDedup", 1, null, { experiments: [{ experiment: "d1", sources: [{ author: "a1", index: 0 }] }] });
|
||||
w.decision("selectDiagnosis", 1, null, { diagnose: true, chosen: ["d1"] });
|
||||
const work = join(dir, "work");
|
||||
cpSync(task.baselineDir, work, { recursive: true });
|
||||
appendFileSync(join(work, "initial.py"), "\n\ndef run_experiment(experiment_id, baseline_dir):\n return [], [], 0.0\n");
|
||||
placeBaselineArtifact(task, baseRun, work);
|
||||
const frozen = { config: checkedFull(fullConfig()), steps: stepList(fullConfig()), facts: task.facts, prompts: null, image: "", seed: 1 };
|
||||
return { s: { dir, task, ledger, frozen } as unknown as Study, work, baseline: saved.version };
|
||||
}
|
||||
|
||||
const dxTask = (kind: "diagnosisImplementation" | "diagnosisRepair" | "diagnosisVerification") =>
|
||||
({ actor: "x", kind, object: "r1-diagnosis", stageId: `r1-${kind}-1` });
|
||||
|
||||
test("诊断实现的接线：存版本时跳过只读的原始产物目录；Jev 对照选中实验的原文和全名指标；工作目录放进原始产物", (t) => {
|
||||
const { s, work, baseline } = codeStudy(t);
|
||||
const w = new Writer(s.ledger);
|
||||
w.open("r1-diagnosisImplementation-1", "diagnosisImplementation", 1, ["x"]);
|
||||
const content = { locations: [{ experiment: "d1", locations: ["initial.py:90-92"] }], concerns: "无", body: "无" };
|
||||
const { known, code } = prepareSubmission(s, dxTask("diagnosisImplementation"), content, 0, work);
|
||||
const saved = (known as { code: ForwardCode }).code;
|
||||
if (!saved.saved) throw new Error("应存成版本");
|
||||
assert.deepEqual(saved.protectedFiles, []);
|
||||
assert.ok(saved.hunks.length > 0);
|
||||
assert.ok(saved.hunks.every((h) => h.path === "initial.py"));
|
||||
assert.match(saved.protocol.body, /### d1 实验甲\n\n动哪一处：第一步\n\n换个做法/);
|
||||
assert.deepEqual(saved.protocol.metrics.map((m) => m.id), ["d1.note"]);
|
||||
assert.deepEqual(code, { object: "r1-diagnosis", version: code!.version, locations: content.locations });
|
||||
// 工作目录里放着的原始产物没有进版本：当前基线到诊断程序的改动里只有 initial.py。
|
||||
const changes = compareVersions(workspaceRepo(s.dir), baseline, code!.version);
|
||||
assert.deepEqual(changes.files.map((f) => f.path), ["initial.py"]);
|
||||
});
|
||||
|
||||
test("诊断实现、修复、核验取材：实现从当前基线出发，修复从上次提交的诊断程序出发，核验和修复的差异都是当前基线到诊断程序；原始产物只给实现和修复", (t) => {
|
||||
const { s, work, baseline } = codeStudy(t);
|
||||
const w = new Writer(s.ledger);
|
||||
const content = { locations: [{ experiment: "d1", locations: ["initial.py:90-92"] }], concerns: "无", body: "无" };
|
||||
w.open("r1-diagnosisImplementation-1", "diagnosisImplementation", 1, ["x"]);
|
||||
w.open("r1-diagnosisRepair-1", "diagnosisImplementation", 1, ["x"]);
|
||||
w.open("r1-diagnosisVerification-1", "diagnosisVerification", 1, ["v"]);
|
||||
assert.deepEqual(taskCode(s, dxTask("diagnosisImplementation")), { changes: null, startVersion: baseline });
|
||||
const { code } = prepareSubmission(s, dxTask("diagnosisImplementation"), content, 0, work);
|
||||
s.ledger.append("code", code!);
|
||||
const changes = compareVersions(workspaceRepo(s.dir), baseline, code!.version);
|
||||
assert.deepEqual(taskCode(s, dxTask("diagnosisRepair")), { changes, startVersion: code!.version });
|
||||
assert.deepEqual(taskCode(s, dxTask("diagnosisVerification")), { changes, startVersion: null });
|
||||
const baseRun = runPaths(s.dir, "base-1").run;
|
||||
assert.equal(taskArtifact(s, dxTask("diagnosisImplementation")), baseRun);
|
||||
assert.equal(taskArtifact(s, dxTask("diagnosisRepair")), baseRun);
|
||||
assert.equal(taskArtifact(s, dxTask("diagnosisVerification")), null);
|
||||
});
|
||||
+217
-13
@@ -4,11 +4,11 @@ import { join } from "node:path";
|
||||
import { test } from "node:test";
|
||||
import { stepList, type ResearchConfig } from "../src/config/index.ts";
|
||||
import type { AppendKind, Content, LedgerRecord } from "../src/ledger/index.ts";
|
||||
import { Board, forwardNotRunEvent, type StudyRecord } from "../src/materials/index.ts";
|
||||
import { Board, diagnosisNotRunEvent, forwardNotRunEvent, type StudyRecord } from "../src/materials/index.ts";
|
||||
import { runBalance } from "../src/rules/index.ts";
|
||||
import { definition, exited } from "./ledger-helpers.ts";
|
||||
import { judged } from "./materials-helpers.ts";
|
||||
import { nextStep } from "../src/flow/next-step.ts";
|
||||
import { diagnosisProgram, nextStep } from "../src/flow/next-step.ts";
|
||||
import type { Json, Submission, SubmissionKind } from "../src/vocab/index.ts";
|
||||
import { fullConfig } from "./config-helpers.ts";
|
||||
|
||||
@@ -217,10 +217,30 @@ function assigned(book: Book): Book {
|
||||
return book.add("decision", { decision: "entries", round: 0, rule: "assignEntries", ruleVersion: 1, params: null, seed: null, input: null, output });
|
||||
}
|
||||
|
||||
test("认领关了：分配；分配做完：开第 1 轮的提案阶段，名单是全部研究者", () => {
|
||||
const DIAGNOSIS_STEPS = [
|
||||
"diagnosisProposal", "diagnosisDedupe", "diagnosisPrediction", "selectDiagnosis",
|
||||
"diagnosisImplementation", "diagnosisRun", "diagnosisSettle",
|
||||
];
|
||||
|
||||
/** 诊断关着（fullConfig 的写法）：第 round 轮开头的诊断七步各记一条按配置跳过（方法.md 第十一部分"步骤开关"）。 */
|
||||
function diagnosisSkipped(book: Book, round: number): Book {
|
||||
for (const step of DIAGNOSIS_STEPS) {
|
||||
assert.deepEqual(nextStep(book.records), { do: "skip", step: `r${round}-${step}`, reason: "配置关掉了诊断" });
|
||||
book.add("event", { event: "skippedByConfig", reason: "配置关掉了诊断", objects: [`r${round}-${step}`] });
|
||||
}
|
||||
return book;
|
||||
}
|
||||
|
||||
/** 第零轮做完、入口分配好，第 1 轮的诊断七步也记完跳过。 */
|
||||
function inRoundOne(book: Book): Book {
|
||||
return diagnosisSkipped(assigned(book), 1);
|
||||
}
|
||||
|
||||
test("认领关了：分配；分配做完：诊断关着，七步各记按配置跳过，再开第 1 轮的提案阶段，名单是全部研究者", () => {
|
||||
const book = afterRoundZero(fullConfig());
|
||||
assert.deepEqual(nextStep(book.records), { do: "assignEntries" });
|
||||
assigned(book);
|
||||
diagnosisSkipped(book, 1);
|
||||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-proposal", stage: "proposal", round: 1, roster: R });
|
||||
});
|
||||
|
||||
@@ -246,7 +266,7 @@ const each = (kind: SubmissionKind, objects: string[]) => R.flatMap((actor) => o
|
||||
|
||||
/** 第 1 轮写到 last 这一步做完为止。 */
|
||||
function roundOne(last: "proposal" | "dedupe" | "firstPrediction" | "critique" | "response" | "freezeProtocol" | "secondPrediction" | "selectBatch"): Book {
|
||||
const book = assigned(afterRoundZero(fullConfig()));
|
||||
const book = inRoundOne(afterRoundZero(fullConfig()));
|
||||
const order = ["proposal", "dedupe", "firstPrediction", "critique", "response", "freezeProtocol", "secondPrediction", "selectBatch"];
|
||||
const steps: Record<string, () => void> = {
|
||||
proposal: () => stage(book, "r1-proposal", "proposal", [["researcher-1", "proposal", null, P1], ["researcher-2", "proposal", null, P2], ["researcher-3", "proposal", null, P3]]),
|
||||
@@ -278,7 +298,7 @@ function roundOne(last: "proposal" | "dedupe" | "firstPrediction" | "critique" |
|
||||
}
|
||||
|
||||
test("提案阶段一人交了两条、两人没交：只派没交的两人，提案没有对象", () => {
|
||||
const book = assigned(afterRoundZero(fullConfig())).open("r1-proposal", "proposal", 1, R).sub("r1-proposal", "researcher-1", "proposal", null, P1);
|
||||
const book = inRoundOne(afterRoundZero(fullConfig())).open("r1-proposal", "proposal", 1, R).sub("r1-proposal", "researcher-1", "proposal", null, P1);
|
||||
book.sub("r1-proposal", "researcher-1", "proposal", null, "r1-researcher-1-2");
|
||||
const tasks = ["researcher-2", "researcher-3"].map((actor) => ({ actor, kind: "proposal", object: null }));
|
||||
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-proposal", tasks });
|
||||
@@ -350,7 +370,7 @@ test("关掉第一次预测：第 1 轮写一条按配置跳过（对象 r1-firs
|
||||
const config = fullConfig();
|
||||
if (config.flow.preset !== "full") throw new Error("测试配置应是完整流程");
|
||||
config.flow.firstPrediction = false;
|
||||
const book = assigned(afterRoundZero(config));
|
||||
const book = inRoundOne(afterRoundZero(config));
|
||||
stage(book, "r1-proposal", "proposal", [["researcher-1", "proposal", null, P1], ["researcher-2", "proposal", null, P2], ["researcher-3", "proposal", null, P3]]);
|
||||
const output = { fresh: [{ candidate: "r1-c1", proposals: [P1] }, { candidate: "r1-c2", proposals: [P2, P3] }], carried: [] };
|
||||
book.add("decision", { decision: "dedup", round: 1, rule: "jevSameChange", ruleVersion: 1, params: null, seed: null, input: null, output });
|
||||
@@ -648,7 +668,8 @@ test("结算和押的因子写完：开收尾；收尾关闭后解封本轮选
|
||||
book.add("unseal", { candidate: "r1-c1", seals: [1] });
|
||||
assert.deepEqual(nextStep(book.records), { do: "unsealChosen", round: 1 });
|
||||
book.add("unseal", { candidate: "r1-c2", seals: [2] });
|
||||
// 余额还有（已扣 5 次）：开第 2 轮。
|
||||
// 余额还有（已扣 5 次）：开第 2 轮，先是诊断七步的按配置跳过。
|
||||
diagnosisSkipped(book, 2);
|
||||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r2-proposal", stage: "proposal", round: 2, roster: R });
|
||||
});
|
||||
|
||||
@@ -679,7 +700,7 @@ test("余额为 0：不开下一轮，走到导出停下，原因写明", () =>
|
||||
assert.deepEqual(nextStep(book.records), { do: "halt", reason: "导出还没写", objects: [] });
|
||||
});
|
||||
|
||||
test("配置关掉消融、又采纳了多因子改法：先记按配置跳过，再在解释停下", () => {
|
||||
test("配置关掉消融、又采纳了多因子改法：⑬–⑰ 整段（含解释）各记按配置跳过，再开收尾", () => {
|
||||
const book = firstVerification(batchTwo(1), true, true);
|
||||
for (const c of C) recordForward(book, c, 1, "valid");
|
||||
book.add("decision", { decision: "adopt", round: 1, rule: "adopt", ruleVersion: 1, params: null, seed: null, input: null, output: { adopted: ADOPTED } });
|
||||
@@ -693,9 +714,192 @@ test("配置关掉消融、又采纳了多因子改法：先记按配置跳过
|
||||
study.config = config as unknown as Json;
|
||||
study.steps = stepList(config) as unknown as Json;
|
||||
const reason = "配置关掉了消融";
|
||||
assert.deepEqual(nextStep(book.records), { do: "skip", step: "r1-listCells", reason });
|
||||
book.add("event", { event: "skippedByConfig", reason, objects: ["r1-listCells"] });
|
||||
const next = nextStep(book.records);
|
||||
assert.equal(next.do, "halt");
|
||||
assert.match((next as { reason: string }).reason, /不交三个数的解释还没写/);
|
||||
// 方法.md 4.3 ⑫：解释是 ⑭，和列格、选格、消融、结算一起不做；没有"解释停下"的情形。
|
||||
for (const step of ["listCells", "explanation", "selectCells", "ablationImplementation", "ablationVerification", "ablationRun", "settle"]) {
|
||||
assert.deepEqual(nextStep(book.records), { do: "skip", step: `r1-${step}`, reason });
|
||||
book.add("event", { event: "skippedByConfig", reason, objects: [`r1-${step}`] });
|
||||
}
|
||||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-wrapUp", stage: "wrapUp", round: 1, roster: R });
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 诊断（任务书 18 第 1.6 节，方法.md 4.3 诊 1–诊 4）：预期动作手写，不调模型、Jev 或 Docker。
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const DX = "r1-diagnosis";
|
||||
const DX_ACTOR = "experimenter-r1-diagnosis";
|
||||
const DX_VERIFIER = "verifier-r1-diagnosis";
|
||||
|
||||
/** 诊断开着的配置：A = 2、D = 4（方法.md 6.11 现行值），消融关；maxRepairs 和运行次数可调。 */
|
||||
function diagnosisConfig(maxRepairs: number, runs: number): ResearchConfig {
|
||||
const config = { ...fullConfig(), maxRepairs, runs };
|
||||
if (config.flow.preset !== "full") throw new Error("测试配置应是完整流程");
|
||||
config.flow = { ...config.flow, diagnosis: { enabled: true, perResearcher: 2, maxExperiments: 4 }, ablation: { enabled: false } };
|
||||
return config;
|
||||
}
|
||||
|
||||
/** 第零轮做完、入口分配好；诊断开着，第 1 轮的诊断从 r1-diagnosisProposal 开始。 */
|
||||
function diagnosisRoundOne(maxRepairs: number, runs: number): Book {
|
||||
return assigned(afterRoundZero(diagnosisConfig(maxRepairs, runs)));
|
||||
}
|
||||
|
||||
const dxTasks = (kind: SubmissionKind) => R.map((actor) => ({ actor, kind, object: null }));
|
||||
|
||||
/** 诊 1、诊 2 阶段关闭，两个实验合并成 d1、d2，诊 3 的决定写下。 */
|
||||
function selected(book: Book, diagnose: boolean, chosen: string[]): Book {
|
||||
for (const [stageId, stage, kind] of [["r1-diagnosisProposal", "diagnosisProposal", "diagnosisProposal"], ["r1-diagnosisPrediction", "diagnosisPrediction", "diagnosisPrediction"]] as const) {
|
||||
book.open(stageId, stage, 1, R);
|
||||
for (const r of R) book.sub(stageId, r, kind, null, null);
|
||||
book.add("stageClose", { stageId });
|
||||
if (kind === "diagnosisProposal") {
|
||||
book.add("decision", { decision: "diagnosisDedup", round: 1, rule: "jevSameExperiment", ruleVersion: 1, params: null, seed: null, input: null, output: { experiments: [] } });
|
||||
}
|
||||
}
|
||||
return book.add("decision", { decision: "selectDiagnosis", round: 1, rule: "valueMinusCost", ruleVersion: 1, params: null, seed: null, input: null, output: { diagnose, chosen } });
|
||||
}
|
||||
|
||||
/** 诊断实现（n = 1）或修复（n ≥ 2）阶段：实验员交一份，关闭。 */
|
||||
function dxWork(book: Book, n: number): Book {
|
||||
const stageId = n === 1 ? "r1-diagnosisImplementation-1" : `r1-diagnosisRepair-${n - 1}`;
|
||||
book.open(stageId, "diagnosisImplementation", 1, [DX_ACTOR]);
|
||||
book.sub(stageId, DX_ACTOR, n === 1 ? "diagnosisImplementation" : "diagnosisRepair", DX, null);
|
||||
return book.add("stageClose", { stageId });
|
||||
}
|
||||
|
||||
/** 第 n 次诊断核验：results 是每个选中实验"是否通过"。 */
|
||||
function dxVerified(book: Book, n: number, results: Record<string, boolean>): Book {
|
||||
const stageId = `r1-diagnosisVerification-${n}`;
|
||||
book.open(stageId, "diagnosisVerification", 1, [DX_VERIFIER]);
|
||||
const content: Submission<"diagnosisVerification"> = {
|
||||
experiments: Object.entries(results).map(([experiment, ok]) => ({ experiment, followed: ok, nothingElse: true, where: ok ? "initial.py:10" : "没找到" })),
|
||||
body: "无",
|
||||
};
|
||||
book.add("submission", { stageId, author: DX_VERIFIER, kind: "diagnosisVerification", target: DX, item: null, version: 1, content, jev: null });
|
||||
return book.add("stageClose", { stageId });
|
||||
}
|
||||
|
||||
/** 一次诊断运行：没启动，或启动了（运行记录不带判分）。 */
|
||||
function dxRun(book: Book, n: number, ending: "valid" | "notStarted"): Book {
|
||||
const run = `r1-diagnosis-${n}`;
|
||||
book.add("runPlanned", { run, purpose: "diagnosis", round: 1, object: DX, definition });
|
||||
if (ending === "notStarted") return book.add("runResult", { run, process: { ending, reason: "Docker 未启动" }, judged: null });
|
||||
book.add("runStarted", { run });
|
||||
return book.add("runResult", { run, process: exited, judged: null });
|
||||
}
|
||||
|
||||
const openProposal = { do: "openStage", stageId: "r1-proposal", stage: "proposal", round: 1, roster: R };
|
||||
|
||||
test("诊断开着：诊 1 开阶段，每人一个任务、没有对象；交齐关闭后判重合并，再开诊断预测，同样每人一个任务", () => {
|
||||
const book = diagnosisRoundOne(1, 13);
|
||||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-diagnosisProposal", stage: "diagnosisProposal", round: 1, roster: R });
|
||||
book.open("r1-diagnosisProposal", "diagnosisProposal", 1, R);
|
||||
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-diagnosisProposal", tasks: dxTasks("diagnosisProposal") });
|
||||
for (const r of R) book.sub("r1-diagnosisProposal", r, "diagnosisProposal", null, null);
|
||||
assert.deepEqual(nextStep(book.records), { do: "closeStage", stageId: "r1-diagnosisProposal" });
|
||||
book.add("stageClose", { stageId: "r1-diagnosisProposal" });
|
||||
assert.deepEqual(nextStep(book.records), { do: "diagnosisDedupe", round: 1 });
|
||||
book.add("decision", { decision: "diagnosisDedup", round: 1, rule: "jevSameExperiment", ruleVersion: 1, params: null, seed: null, input: null, output: { experiments: [] } });
|
||||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-diagnosisPrediction", stage: "diagnosisPrediction", round: 1, roster: R });
|
||||
book.open("r1-diagnosisPrediction", "diagnosisPrediction", 1, R);
|
||||
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-diagnosisPrediction", tasks: dxTasks("diagnosisPrediction") });
|
||||
for (const r of R) book.sub("r1-diagnosisPrediction", r, "diagnosisPrediction", null, null);
|
||||
book.add("stageClose", { stageId: "r1-diagnosisPrediction" });
|
||||
assert.deepEqual(nextStep(book.records), { do: "selectDiagnosis", round: 1 });
|
||||
});
|
||||
|
||||
test("余额不到 2 次：诊 1–诊 4 都不做，不开阶段、不写记录，直接开提案（运行 2 次，基线用掉 1 次，余额 1）", () => {
|
||||
const book = diagnosisRoundOne(1, 2);
|
||||
assert.deepEqual(nextStep(book.records), openProposal);
|
||||
});
|
||||
|
||||
test("诊 3 定了不诊断：诊 4 的几步都不做，直接开提案；决定记录照写", () => {
|
||||
const book = selected(diagnosisRoundOne(1, 13), false, []);
|
||||
assert.deepEqual(nextStep(book.records), openProposal);
|
||||
});
|
||||
|
||||
test("诊断实现与核验：一位实验员对本轮诊断，一位核验员逐个实验交结果；全通过就运行，没启动起新编号，已登记没结果要先恢复", () => {
|
||||
const book = selected(diagnosisRoundOne(1, 13), true, ["d1", "d2"]);
|
||||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-diagnosisImplementation-1", stage: "diagnosisImplementation", round: 1, roster: [DX_ACTOR] });
|
||||
book.open("r1-diagnosisImplementation-1", "diagnosisImplementation", 1, [DX_ACTOR]);
|
||||
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-diagnosisImplementation-1", tasks: [{ actor: DX_ACTOR, kind: "diagnosisImplementation", object: DX }] });
|
||||
book.sub("r1-diagnosisImplementation-1", DX_ACTOR, "diagnosisImplementation", DX, null);
|
||||
book.add("stageClose", { stageId: "r1-diagnosisImplementation-1" });
|
||||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-diagnosisVerification-1", stage: "diagnosisVerification", round: 1, roster: [DX_VERIFIER] });
|
||||
dxVerified(book, 1, { d1: true, d2: true });
|
||||
assert.deepEqual(nextStep(book.records), { do: "diagnosisRun", round: 1, run: "r1-diagnosis-1" });
|
||||
dxRun(book, 1, "notStarted");
|
||||
assert.deepEqual(nextStep(book.records), { do: "diagnosisRun", round: 1, run: "r1-diagnosis-2" });
|
||||
assert.deepEqual(runBalance(13, new Board(book.records).runStates()), { charged: 1, reserved: 0, balance: 12 });
|
||||
book.add("runPlanned", { run: "r1-diagnosis-2", purpose: "diagnosis", round: 1, object: DX, definition });
|
||||
assert.throws(() => nextStep(book.records), /诊断运行 r1-diagnosis-2 登记了还没有结果：恢复入口应当已经核对过/);
|
||||
book.add("runStarted", { run: "r1-diagnosis-2" });
|
||||
book.add("runResult", { run: "r1-diagnosis-2", process: exited, judged: null });
|
||||
assert.deepEqual(nextStep(book.records), { do: "diagnosisSettle", round: 1 });
|
||||
// 没启动的那次退还名额：基线 1 次、诊断 1 次，诊断运行只扣一次。
|
||||
assert.deepEqual(runBalance(13, new Board(book.records).runStates()), { charged: 2, reserved: 0, balance: 11 });
|
||||
book.add("diagnosisSettlement", { round: 1, result: null });
|
||||
assert.deepEqual(nextStep(book.records), openProposal);
|
||||
});
|
||||
|
||||
test("核验有实验不通过：只退回修复，只重验一次；用完修复仍不通过的实验拿掉，跑的只是剩下的实验", () => {
|
||||
const book = selected(diagnosisRoundOne(1, 13), true, ["d1", "d2"]);
|
||||
dxWork(book, 1);
|
||||
dxVerified(book, 1, { d1: true, d2: false });
|
||||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-diagnosisRepair-1", stage: "diagnosisImplementation", round: 1, roster: [DX_ACTOR] });
|
||||
book.open("r1-diagnosisRepair-1", "diagnosisImplementation", 1, [DX_ACTOR]);
|
||||
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-diagnosisRepair-1", tasks: [{ actor: DX_ACTOR, kind: "diagnosisRepair", object: DX }] });
|
||||
book.sub("r1-diagnosisRepair-1", DX_ACTOR, "diagnosisRepair", DX, null);
|
||||
book.add("stageClose", { stageId: "r1-diagnosisRepair-1" });
|
||||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-diagnosisVerification-2", stage: "diagnosisVerification", round: 1, roster: [DX_VERIFIER] });
|
||||
dxVerified(book, 2, { d1: true, d2: false });
|
||||
// 修复次数上限 1 已用完：d2 拿掉，d1 照跑；没有"不跑"事件。
|
||||
assert.deepEqual(nextStep(book.records), { do: "diagnosisRun", round: 1, run: "r1-diagnosis-1" });
|
||||
const dropped = diagnosisProgram(new Board(book.records), 1, 1);
|
||||
assert.deepEqual(dropped.experiments, ["d1"]);
|
||||
assert.deepEqual(dropped.dropped.map((d) => d.experiment), ["d2"]);
|
||||
assert.match(dropped.dropped[0]!.reason, /第 2 次核验仍不通过，修复次数上限 1 已用完：没照写法做；没找到/);
|
||||
});
|
||||
|
||||
test("修复后上次通过的实验又不通过：以最近一次核验为准，修复前通过过也不能跑", () => {
|
||||
const book = selected(diagnosisRoundOne(1, 13), true, ["d1", "d2"]);
|
||||
dxWork(book, 1);
|
||||
dxVerified(book, 1, { d1: true, d2: false });
|
||||
dxWork(book, 2);
|
||||
dxVerified(book, 2, { d1: false, d2: true });
|
||||
assert.deepEqual(diagnosisProgram(new Board(book.records), 1, 1).experiments, ["d2"]);
|
||||
assert.deepEqual(nextStep(book.records), { do: "diagnosisRun", round: 1, run: "r1-diagnosis-1" });
|
||||
});
|
||||
|
||||
test("maxRepairs = 0：第一次核验不通过的实验就拿掉，不开修复阶段", () => {
|
||||
const book = selected(diagnosisRoundOne(0, 13), true, ["d1", "d2"]);
|
||||
dxWork(book, 1);
|
||||
dxVerified(book, 1, { d1: false, d2: true });
|
||||
assert.deepEqual(nextStep(book.records), { do: "diagnosisRun", round: 1, run: "r1-diagnosis-1" });
|
||||
assert.deepEqual(diagnosisProgram(new Board(book.records), 1, 0).experiments, ["d2"]);
|
||||
});
|
||||
|
||||
test("选中的实验一个都不剩：写不跑事件（一次），不登记运行、不扣次，不结算，直接开提案", () => {
|
||||
const book = selected(diagnosisRoundOne(1, 13), true, ["d1", "d2"]);
|
||||
dxWork(book, 1);
|
||||
dxVerified(book, 1, { d1: false, d2: false });
|
||||
dxWork(book, 2);
|
||||
dxVerified(book, 2, { d1: false, d2: false });
|
||||
const next = nextStep(book.records);
|
||||
assert.equal(next.do, "diagnosisNotRun");
|
||||
if (next.do !== "diagnosisNotRun") throw new Error("应写不跑事件");
|
||||
assert.equal(next.round, 1);
|
||||
assert.match(next.reason, /选中的实验第 2 次核验都不通过，修复次数上限 1 已用完：d1：.*；d2：/);
|
||||
book.add("event", diagnosisNotRunEvent(1, next.reason));
|
||||
assert.deepEqual(nextStep(book.records), openProposal);
|
||||
assert.deepEqual(runBalance(13, new Board(book.records).runStates()), { charged: 1, reserved: 0, balance: 12 });
|
||||
});
|
||||
|
||||
test("诊断运行扣次以后：余额只算本轮开始之前的运行，诊断这一段的后几步不会算成不诊断", () => {
|
||||
// 运行 3 次：本轮开始前余额 2，诊断运行扣 1 次后余额 1。诊断这一段的后几步照常走，不会因余额变小算成"不诊断"。
|
||||
const book = selected(diagnosisRoundOne(1, 3), true, ["d1"]);
|
||||
dxWork(book, 1);
|
||||
dxVerified(book, 1, { d1: true });
|
||||
dxRun(book, 1, "valid");
|
||||
assert.equal(runBalance(3, new Board(book.records).runStates()).balance, 1);
|
||||
assert.deepEqual(nextStep(book.records), { do: "diagnosisSettle", round: 1 });
|
||||
});
|
||||
|
||||
+59
-1
@@ -3,7 +3,17 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { test, type TestContext } from "node:test";
|
||||
import { checkHunk, checkProtocol, codeHunks, judgeSameChange, type FrozenProtocol, type ProtocolRules } from "../src/jev/index.ts";
|
||||
import {
|
||||
checkExperiment,
|
||||
checkHunk,
|
||||
checkProtocol,
|
||||
codeHunks,
|
||||
judgeSameChange,
|
||||
judgeSameExperiment,
|
||||
type DiagnosisExperiment,
|
||||
type FrozenProtocol,
|
||||
type ProtocolRules,
|
||||
} from "../src/jev/index.ts";
|
||||
import { hash, type Json } from "../src/vocab/index.ts";
|
||||
import { compareVersions, createRepository, saveVersion } from "../src/workspace/index.ts";
|
||||
|
||||
@@ -307,3 +317,51 @@ test("把 workspace 的比较结果切成差异块，每块从 @@ 行起", (t) =
|
||||
assert.equal(hunks[0]?.text, "@@ -1,5 +1,5 @@\n line 1\n-line 2\n+changed 2\n line 3\n line 4\n line 5");
|
||||
assert.equal(hunks[2]?.text, "@@ -0,0 +1 @@\n+x = 1");
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 诊断实验的判重和冻结检查（方法.md 4.3 诊 1、诊 2）
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const EXPERIMENT: DiagnosisExperiment = {
|
||||
name: "拿掉收尾",
|
||||
spot: "solve.py 的收尾修复",
|
||||
body: "拿掉收尾修复，直接交出原始答案",
|
||||
metrics: [{ id: "raw-score", meaning: "收尾之前的分数", unit: "无", direction: "maximize", howMeasured: "收尾之前的分数", missing: "recordMissing" }],
|
||||
};
|
||||
|
||||
test("诊断实验判重：一对一次请求，只给名称、动哪一处、怎么动；阈值暂用判重的 0.3", async (t) => {
|
||||
const sent = fakeJev(t, answerAll(() => 0.31));
|
||||
const other = { name: "强起点", spot: "solve.py 的起点", body: "换成更强的起点" };
|
||||
const j = await judgeSameExperiment("jev-x", EXPERIMENT, other);
|
||||
assert.equal(sent.length, 1);
|
||||
assert.deepEqual(Object.keys(sent[0]!.body.questions), ["same_experiment"]);
|
||||
assert.deepEqual(sent[0]!.body.state.experiment_a, { name: "拿掉收尾", spot: "solve.py 的收尾修复", body: "拿掉收尾修复，直接交出原始答案" });
|
||||
assert.deepEqual(sent[0]!.body.state.experiment_b, other);
|
||||
assert.deepEqual([j.question, j.threshold, j.yes, j.requestId], ["same_experiment", 0.3, true, "req_1"]);
|
||||
});
|
||||
|
||||
test("诊断实验判重：概率等于阈值判否", async (t) => {
|
||||
fakeJev(t, answerAll(() => 0.3));
|
||||
assert.equal((await judgeSameExperiment("jev-x", EXPERIMENT, EXPERIMENT)).yes, false);
|
||||
});
|
||||
|
||||
test("诊断实验冻结检查：三问各一次请求，材料相同，问法是诊断的变体；有一问判是就标出", async (t) => {
|
||||
const sent = fakeJev(t, answerAll((name) => (name === "changes_measurement" ? 0.22 : 0.1)));
|
||||
const result = await checkExperiment("jev-x", EXPERIMENT);
|
||||
assert.deepEqual(
|
||||
result.judgments.map((j) => [j.question, j.threshold, j.yes]),
|
||||
[["self_contradictory", 0.5, false], ["changes_measurement", 0.21, true], ["too_vague", 0.78, false]],
|
||||
);
|
||||
assert.equal(result.flagged, true);
|
||||
assert.deepEqual(sent.map((s) => Object.keys(s.body.questions)), [["self_contradictory"], ["changes_measurement"], ["too_vague"]]);
|
||||
for (const s of sent) assert.deepEqual(s.body.state, sent[0]!.body.state);
|
||||
assert.deepEqual(sent[0]!.body.state.experiment, {
|
||||
name: "拿掉收尾",
|
||||
spot: "solve.py 的收尾修复",
|
||||
body: "拿掉收尾修复，直接交出原始答案",
|
||||
metrics: [{ id: "raw-score", howMeasured: "收尾之前的分数", missing: "recordMissing" }],
|
||||
});
|
||||
// 材料说明和问法都是诊断的变体，哈希和冻结协议同名三问的不同。
|
||||
const protocol = await checkProtocol("jev-x", PROTOCOL);
|
||||
result.judgments.forEach((j, i) => assert.notEqual(j.questionHash, protocol.judgments[i]!.questionHash, j.question));
|
||||
});
|
||||
|
||||
@@ -80,7 +80,7 @@ export function closeStage(ledger: Ledger, stageId: string): number {
|
||||
}
|
||||
|
||||
/** 登记一次运行。 */
|
||||
export function planRun(ledger: Ledger, run: string, purpose: "baseline" | "forward" | "ablation", round: number): number {
|
||||
export function planRun(ledger: Ledger, run: string, purpose: "baseline" | "forward" | "ablation" | "diagnosis", round: number): number {
|
||||
return ledger.append("runPlanned", { run, purpose, round, object: purpose === "baseline" ? null : `obj-${run}`, definition }).seq;
|
||||
}
|
||||
|
||||
|
||||
@@ -16,12 +16,13 @@ import {
|
||||
wrapUp,
|
||||
} from "./ledger-helpers.ts";
|
||||
|
||||
// 写入时由 ledger 拒绝的四种情况（docs/结构.md 第四部分"写入时由 ledger 拒绝的几种情况"），每种一条拒收、一条对应的正常情况。
|
||||
// 写入时由 ledger 拒绝的五种情况（docs/结构.md 第四部分"写入时由 ledger 拒绝的几种情况"），每种一条拒收、一条对应的正常情况。
|
||||
// 预期值来自文字：
|
||||
// 1. 只收开着的阶段（有开阶段记录、没有关阶段记录）的提交，别的拒收；
|
||||
// 2. 要替换的那一份不是本人在这个阶段、对同一个对象的最新一版 → 拒收；实现、修复、核验、消融实现、消融核验不能替换（方法.md 第五部分）；
|
||||
// 3. 运行结果写在"确认启动"之前 → 拒收；"没启动"的结果不需要启动记录（方法.md 第七部分）；
|
||||
// 4. 这一轮消融已经有格确认启动了，才来关这一轮的解释阶段 → 拒收（方法.md 4.3 ⑭）。
|
||||
// 4. 这一轮消融已经有格确认启动了，才来关这一轮的解释阶段 → 拒收（方法.md 4.3 ⑭）；
|
||||
// 5. 这一轮的诊断运行已经确认启动了，才来关这一轮的诊断预测阶段 → 拒收（方法.md 4.3 诊 2）；诊断实现、修复、核验和实现、核验一样不能替换。
|
||||
|
||||
function submitWrapUp(ledger: Ledger, stageId: string, author: string, version: number, body = "收尾") {
|
||||
return ledger.append("submission", { stageId, author, target: null, item: null, version, jev: null, kind: "wrapUp", content: wrapUp(body) });
|
||||
@@ -156,3 +157,53 @@ test("拒收 4 的正常情况：只登记没启动、别的轮的消融启动
|
||||
closeStage(ledger, "wrap");
|
||||
assert.equal(ledger.stageClosed("wrap"), true);
|
||||
});
|
||||
|
||||
test("拒收 5：这一轮的诊断运行已经确认启动，就不能关这一轮的诊断预测阶段", (t) => {
|
||||
const { ledger } = freshLedger(t);
|
||||
openStage(ledger, "dpred", "diagnosisPrediction", 2);
|
||||
planRun(ledger, "diagrun", "diagnosis", 2);
|
||||
ledger.append("runStarted", { run: "diagrun" });
|
||||
assert.throws(() => closeStage(ledger, "dpred"), /诊断预测冻结必须早于诊断运行启动/);
|
||||
assert.equal(ledger.stageClosed("dpred"), false);
|
||||
});
|
||||
|
||||
test("拒收 5 的正常情况：只登记没启动、别的轮的诊断启动、本轮消融或正向启动，都不挡关诊断预测阶段；消融启动也不被诊断运行连累", (t) => {
|
||||
const { ledger } = freshLedger(t);
|
||||
openStage(ledger, "dpred", "diagnosisPrediction", 2);
|
||||
planRun(ledger, "diag-planned", "diagnosis", 2); // 只登记，还没确认启动
|
||||
planRun(ledger, "diag-round1", "diagnosis", 1);
|
||||
ledger.append("runStarted", { run: "diag-round1" });
|
||||
planRun(ledger, "fwd-round2", "forward", 2);
|
||||
ledger.append("runStarted", { run: "fwd-round2" });
|
||||
closeStage(ledger, "dpred");
|
||||
assert.equal(ledger.stageClosed("dpred"), true);
|
||||
// 诊断运行启动后，解释阶段照旧关得掉：两种用途各管各的。
|
||||
openStage(ledger, "expl", "explanation", 2);
|
||||
planRun(ledger, "diag-round2", "diagnosis", 2);
|
||||
ledger.append("runStarted", { run: "diag-round2" });
|
||||
closeStage(ledger, "expl");
|
||||
assert.equal(ledger.stageClosed("expl"), true);
|
||||
});
|
||||
|
||||
test("拒收 2：诊断实现、诊断修复、诊断核验只收第一版，不能替换；诊断预测可以替换", (t) => {
|
||||
const { ledger } = freshLedger(t);
|
||||
openStage(ledger, "dimpl", "diagnosisImplementation", 1, ["e1"]);
|
||||
openStage(ledger, "dver", "diagnosisVerification", 1, ["v1"]);
|
||||
openStage(ledger, "dpred", "diagnosisPrediction", 1);
|
||||
const base = { author: "e1", target: "r1-diagnosis", item: null };
|
||||
const code = { locations: [{ experiment: "d1", locations: ["solve.py:3-9"] }], concerns: "无", body: "无" };
|
||||
const cases = [
|
||||
{ ...base, stageId: "dimpl", kind: "diagnosisImplementation" as const, content: code },
|
||||
{ ...base, stageId: "dimpl", author: "e2", kind: "diagnosisRepair" as const, content: code },
|
||||
{ ...base, author: "v1", stageId: "dver", kind: "diagnosisVerification" as const, content: { experiments: [], body: "无" } },
|
||||
];
|
||||
for (const c of cases) {
|
||||
ledger.append("submission", { ...c, version: 1, jev: null });
|
||||
assert.throws(() => ledger.append("submission", { ...c, version: 2, jev: null }), /不能替换/, c.kind);
|
||||
}
|
||||
const slot = { stageId: "dpred", author: "r1", target: null, item: null, kind: "diagnosisPrediction" as const };
|
||||
const content = { predictions: [{ explanation: "r1-r1-dx1", experiment: "d1", range: range(1, 2, 3) }], body: "理由" };
|
||||
ledger.append("submission", { ...slot, version: 1, jev: null, content });
|
||||
ledger.append("submission", { ...slot, version: 2, jev: null, content });
|
||||
assert.equal(ledger.latestSubmission(slot)?.content.version, 2);
|
||||
});
|
||||
|
||||
@@ -438,6 +438,16 @@ test("关掉再打开：读出的记录不变，序号接着往下排", (t) => {
|
||||
assert.equal(reopened.append("runStarted", { run: "run1" }).seq, 9);
|
||||
});
|
||||
|
||||
test("唯一约束：诊断结算按轮次，同一轮两次报错；同一轮的消融结算另算，不受它影响", (t) => {
|
||||
const { ledger, path } = freshLedger(t);
|
||||
ledger.append("diagnosisSettlement", { round: 1, result: { experiments: [] } });
|
||||
assert.throws(() => ledger.append("diagnosisSettlement", { round: 1, result: { experiments: [] } }), /diagnosis_settlement_once/);
|
||||
ledger.append("diagnosisSettlement", { round: 2, result: { experiments: [] } });
|
||||
ledger.append("settlement", { round: 1, result: { cells: {} } });
|
||||
assert.throws(() => ledger.append("diagnosisSettlement", { round: 3, run: "x", result: {} } as never), /diagnosisSettlement/);
|
||||
assert.equal(countRows(path), 3);
|
||||
});
|
||||
|
||||
test("打开：表结构版本不对报错；不是空文件也不是账本的库报错", (t) => {
|
||||
const dir = tempDir(t);
|
||||
const wrongVersion = join(dir, "v99.db");
|
||||
@@ -446,12 +456,12 @@ test("打开：表结构版本不对报错；不是空文件也不是账本的
|
||||
db.close();
|
||||
assert.throws(() => Ledger.open(wrongVersion), /版本是 99/);
|
||||
|
||||
// 版本 3 是退回还没有对象字段时的账本，不迁移，打开报错。
|
||||
const previous = join(dir, "v3.db");
|
||||
const v3 = new DatabaseSync(previous);
|
||||
v3.exec("PRAGMA user_version = 3");
|
||||
v3.close();
|
||||
assert.throws(() => Ledger.open(previous), /版本是 3.*只认版本 4/s);
|
||||
// 版本 4 是还没有诊断结算和诊断运行用途时的账本，不迁移，打开报错。
|
||||
const previous = join(dir, "v4.db");
|
||||
const v4 = new DatabaseSync(previous);
|
||||
v4.exec("PRAGMA user_version = 4");
|
||||
v4.close();
|
||||
assert.throws(() => Ledger.open(previous), /版本是 4.*只认版本 5/s);
|
||||
|
||||
const foreign = join(dir, "foreign.db");
|
||||
const other = new DatabaseSync(foreign);
|
||||
|
||||
@@ -0,0 +1,337 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { test, type TestContext } from "node:test";
|
||||
import { Ledger } from "../src/ledger/index.ts";
|
||||
import {
|
||||
Board,
|
||||
buildMaterial,
|
||||
type CodeChanges,
|
||||
diagnosisExplanationId,
|
||||
diagnosisId,
|
||||
diagnosisNotRunEvent,
|
||||
type DiagnosisSettlement,
|
||||
type StudyFacts,
|
||||
type TaskMaterial,
|
||||
} from "../src/materials/index.ts";
|
||||
import { selectDiagnosis, settleDiagnosis, type DiagnosisRuleInput, type SettleDiagnosisInput } from "../src/rules/index.ts";
|
||||
import type { SubmissionKind } from "../src/vocab/index.ts";
|
||||
import { freshLedger } from "./ledger-helpers.ts";
|
||||
import { batch, CHANGES_C1, everything, judged, material, NONE, P_A1, prompts, protocolOf, r, study, TABLE_C1, Writer } from "./materials-helpers.ts";
|
||||
|
||||
// 诊断的材料（任务书 18 第 1.5 节、方法.md 第三部分、4.3 诊 1–诊 4）：在真实账本里写一轮诊断，看每个任务该有的有、不该有的没有。
|
||||
// 研究者 a1、a2，实验员 x1，核验员 v1。基线 0.96。三个合并后的诊断实验：d1（a1、a2 的"强起点"合并）、d2（a1 的）、d3（a2 的第二个）。
|
||||
// a1 的诊断预测数 7.771…、a2 的 8.881…是好认的数：任何别人的材料里都不该出现。
|
||||
// 选实验的决定、结算都用 rules 的函数算（不手写数）；这里只查材料怎么读它们。
|
||||
|
||||
const SEALED = { a1: ["7.771", "7.772", "7.773"], a2: ["8.881", "8.882", "8.883"] };
|
||||
const D = diagnosisId(1);
|
||||
const headings = (m: TaskMaterial) => m.sections.map((s) => s.heading);
|
||||
const has = (m: TaskMaterial, text: string) => assert.ok(everything(m).includes(text), `材料里应有"${text}"`);
|
||||
const lacks = (m: TaskMaterial, text: string) => assert.ok(!everything(m).includes(text), `材料里不该有"${text}"`);
|
||||
|
||||
const experiment = (name: string, spot: string) => ({ name, spot, body: `${name}：${spot}上怎么动`, metrics: [] });
|
||||
const gapExperiment = { ...experiment("拿掉收尾", "solve.py 收尾"), metrics: [{ id: "gap", meaning: "间隙", unit: "无", direction: "minimize" as const, howMeasured: "最后量一次", missing: "recordMissing" as const }] };
|
||||
|
||||
/** 两条解释对三个实验的三个数：a1 说 d1 能涨很多，a2 说 d1 涨不了；d2、d3 两人差不多。 */
|
||||
const ranges = {
|
||||
[diagnosisExplanationId(1, "a1", 1)]: { author: "a1", d1: r(1.1, 1.25, 1.3), d2: r(0.95, 1.0, 1.05), d3: r(0.95, 1.0, 1.05) },
|
||||
[diagnosisExplanationId(1, "a2", 1)]: { author: "a2", d1: r(0.9, 0.95, 0.99), d2: r(0.95, 1.0, 1.05), d3: r(0.95, 1.0, 1.05) },
|
||||
};
|
||||
const ruleInput = (balance: number, same: boolean): DiagnosisRuleInput => ({
|
||||
baseline: 0.96,
|
||||
direction: "maximize",
|
||||
lambda: 0.1,
|
||||
experiments: ["d1", "d2", "d3"],
|
||||
explanations: Object.entries(ranges).map(([id, x]) => ({
|
||||
id,
|
||||
author: x.author,
|
||||
experiments: (["d1", "d2", "d3"] as const).map((e) => ({ experiment: e, range: same ? ranges[diagnosisExplanationId(1, "a1", 1)]![e] : x[e] })),
|
||||
})),
|
||||
balance,
|
||||
batchCap: 1,
|
||||
u: 0.5,
|
||||
decay: 0.5,
|
||||
maxExperiments: 2,
|
||||
});
|
||||
|
||||
type Step = "diagProposalOpen" | "diagProposalClosed" | "diagPredictionOpen" | "diagPredictionClosed" | "selected" | "implemented" | "verifiedFailed" | "repairOpen" | "run" | "settled";
|
||||
const ORDER: Step[] = ["diagProposalOpen", "diagProposalClosed", "diagPredictionOpen", "diagPredictionClosed", "selected", "implemented", "verifiedFailed", "repairOpen", "run", "settled"];
|
||||
|
||||
/** 第 1 轮诊断写到某一步（含）为止。选中的是净值最大的两个实验；d 里验证不通过的那个在结算里记证据不足。 */
|
||||
function diagnosisStudy(t: TestContext, upTo: Step): { l: Ledger; w: Writer; chosen: string[] } {
|
||||
const { ledger: l } = freshLedger(t);
|
||||
const w = new Writer(l);
|
||||
const reached = (s: Step) => ORDER.indexOf(upTo) >= ORDER.indexOf(s);
|
||||
l.append("code", { object: "base", version: "v-base", locations: null });
|
||||
w.run("run-0", "baseline", 0, null, judged(0.96, "main"));
|
||||
w.open("s1-dp", "diagnosisProposal", 1, ["a1", "a2"]);
|
||||
w.sub("s1-dp", "a1", "diagnosisProposal", null, null, {
|
||||
explanations: [{ body: "甲的诊断解释：卡在起点 UNIQUE-A1" }],
|
||||
experiments: [experiment("强起点", "solve.py 起点"), gapExperiment],
|
||||
});
|
||||
if (!reached("diagProposalClosed")) return { l, w, chosen: [] };
|
||||
w.sub("s1-dp", "a2", "diagnosisProposal", null, null, {
|
||||
explanations: [{ body: "乙的诊断解释：卡在收尾 UNIQUE-A2" }],
|
||||
experiments: [experiment("强起点", "solve.py 起点（乙的写法）"), experiment("放宽间隙", "solve.py 间隙限制")],
|
||||
});
|
||||
w.close("s1-dp");
|
||||
w.decision("diagnosisDedup", 1, null, {
|
||||
experiments: [
|
||||
{ experiment: "d1", sources: [{ author: "a1", index: 0 }, { author: "a2", index: 0 }] },
|
||||
{ experiment: "d2", sources: [{ author: "a1", index: 1 }] },
|
||||
{ experiment: "d3", sources: [{ author: "a2", index: 1 }] },
|
||||
],
|
||||
});
|
||||
w.open("s1-dpred", "diagnosisPrediction", 1, ["a1", "a2"]);
|
||||
const pred = (author: "a1" | "a2", numbers: string[]) =>
|
||||
(["d1", "d2", "d3"] as const).map((e) => ({ explanation: diagnosisExplanationId(1, author, 1), experiment: e, range: e === "d1" ? r(...(numbers.map(Number) as [number, number, number])) : ranges[diagnosisExplanationId(1, author, 1)]![e] }));
|
||||
w.sub("s1-dpred", "a1", "diagnosisPrediction", null, null, { predictions: pred("a1", SEALED.a1), body: "甲的预测理由" });
|
||||
if (!reached("diagPredictionClosed")) return { l, w, chosen: [] };
|
||||
w.sub("s1-dpred", "a2", "diagnosisPrediction", null, null, { predictions: pred("a2", SEALED.a2), body: "乙的预测理由" });
|
||||
w.close("s1-dpred");
|
||||
const input = ruleInput(9, false);
|
||||
const decision = selectDiagnosis(input);
|
||||
assert.equal(decision.diagnose, true);
|
||||
const chosen = decision.chosen;
|
||||
assert.equal(chosen.length, 2);
|
||||
if (!reached("selected")) return { l, w, chosen };
|
||||
w.decision("selectDiagnosis", 1, input, decision);
|
||||
w.open("s1-dimpl", "diagnosisImplementation", 1, ["x1"]);
|
||||
const table = chosen.map((experiment, i) => ({ experiment, locations: [`solve.py:${10 + 10 * i}-${19 + 10 * i}`] }));
|
||||
w.sub("s1-dimpl", "x1", "diagnosisImplementation", D, null, { locations: table, concerns: "收尾里有个旧开关", body: "诊断实现说明 UNIQUE-X1" });
|
||||
w.close("s1-dimpl");
|
||||
l.append("code", { object: D, version: "v-diag", locations: table as never });
|
||||
if (!reached("implemented")) return { l, w, chosen };
|
||||
w.open("s1-dver", "diagnosisVerification", 1, ["v1"]);
|
||||
w.sub("s1-dver", "v1", "diagnosisVerification", D, null, {
|
||||
experiments: chosen.map((experiment, i) => ({ experiment, followed: i === 0, nothingElse: true, where: `solve.py:${10 + 10 * i}` })),
|
||||
body: "核验员：第二个实验没照写法做",
|
||||
});
|
||||
w.close("s1-dver");
|
||||
if (!reached("repairOpen")) return { l, w, chosen };
|
||||
w.open("s1-dimpl2", "diagnosisImplementation", 1, ["x1"]);
|
||||
if (!reached("run")) return { l, w, chosen };
|
||||
w.run("run-1", "diagnosis", 1, D, null); // 诊断运行的记录里没有判分：逐个实验的结果在结算里
|
||||
if (!reached("settled")) return { l, w, chosen };
|
||||
const settleInput: SettleDiagnosisInput = {
|
||||
baseline: 0.96,
|
||||
direction: "maximize",
|
||||
epsilon: 0.01,
|
||||
lambda: 0.1,
|
||||
experiments: ["d1", "d2", "d3"].map((experiment) => ({
|
||||
experiment,
|
||||
outcome: experiment === chosen[0] ? { kind: "valid", score: 1.1 } : experiment === chosen[1] ? { kind: "insufficient" } : { kind: "untested" },
|
||||
})),
|
||||
explanations: ruleInput(9, false).explanations,
|
||||
};
|
||||
const settlement: DiagnosisSettlement = {
|
||||
run: "run-1",
|
||||
judged: {
|
||||
experiments: [
|
||||
{
|
||||
id: chosen[0]!, ending: "finished", wallSeconds: { status: "measured", value: 31.5 }, verdict: "valid", verdictReason: "校验器判合法",
|
||||
score: { status: "measured", value: 1.1 }, metrics: { "sum-of-radii": { status: "measured", value: 1.1 }, [`${chosen[0]}.gap`]: { status: "measured", value: 0.02 } }, extra: null,
|
||||
},
|
||||
],
|
||||
},
|
||||
dropped: [{ experiment: chosen[1]!, reason: "第二个实验两次核验都没落实" }],
|
||||
input: settleInput,
|
||||
output: settleDiagnosis(settleInput),
|
||||
};
|
||||
l.append("diagnosisSettlement", { round: 1, result: settlement as never });
|
||||
return { l, w, chosen };
|
||||
}
|
||||
|
||||
const mat = (l: Ledger, kind: SubmissionKind, actor: string, stageId: string, object: string | null, extra: { changes: CodeChanges | null; artifacts: Record<string, Record<string, string>> | null } = NONE, facts: StudyFacts = study) =>
|
||||
buildMaterial({ kind, actor, stageId, object, ledger: l, study: facts, prompts, codeRoot: "/code", ...extra });
|
||||
|
||||
test("诊 1：基线代码、基线那次运行的结果和答案、没有别人这一轮的诊断；上限取自配置（A、M）", (t) => {
|
||||
const { l } = diagnosisStudy(t, "diagProposalOpen");
|
||||
const m = mat(l, "diagnosisProposal", "a2", "s1-dp", null, { changes: null, artifacts: { "run-0": { "solution.json": "{\"centers\": []}" } } });
|
||||
assert.deepEqual(m.checklist, {
|
||||
kind: "diagnosisProposal",
|
||||
maxExplanations: 2,
|
||||
maxExperiments: 2,
|
||||
takenMetricIds: ["sum-of-radii", "wall-seconds", "answer-path", "safeguard-triggers"],
|
||||
});
|
||||
assert.deepEqual(m.readable, [{ version: "v-base", label: "本批基线", dir: "/code/v-base" }]);
|
||||
has(m, "运行 run-0：任务给的原始代码；算数（校验器判合法）；分数 0.96");
|
||||
has(m, "run-0/solution.json");
|
||||
assert.equal(m.records.find((x) => x.id === "run-0/solution.json")!.text, "{\"centers\": []}");
|
||||
assert.ok(headings(m).includes("上一轮的收尾"));
|
||||
assert.ok(!headings(m).includes("本轮诊断结果"), "诊断这一段还没走完，不放本轮诊断结果");
|
||||
lacks(m, "UNIQUE-A1"); // a1 在同一阶段已经交了，阶段没关，别人读不到
|
||||
lacks(m, "强起点");
|
||||
});
|
||||
|
||||
test("诊 2：合并后的实验标明作者、全部诊断解释、自己的解释另列并给对号清单；看不到别人在这一步交的数", (t) => {
|
||||
const { l } = diagnosisStudy(t, "diagPredictionOpen");
|
||||
const m = mat(l, "diagnosisPrediction", "a2", "s1-dpred", null);
|
||||
assert.deepEqual(m.checklist, { kind: "diagnosisPrediction", explanations: ["r1-a2-dx1"], experiments: ["d1", "d2", "d3"] });
|
||||
has(m, "#### d1 强起点（作者 a1、a2）");
|
||||
has(m, "动哪一处：solve.py 起点\n"); // 合并后的正文取作者里编号最小的那一份（a1）
|
||||
lacks(m, "solve.py 起点（乙的写法）");
|
||||
has(m, "#### d3 放宽间隙（作者 a2）");
|
||||
has(m, "UNIQUE-A1");
|
||||
has(m, "UNIQUE-A2");
|
||||
assert.ok(m.sections.some((s) => s.heading === "你的诊断解释" && s.text.includes("r1-a2-dx1")));
|
||||
for (const x of SEALED.a1) lacks(m, x); // a1 已经交了诊断预测，阶段还开着
|
||||
});
|
||||
|
||||
test("诊断预测的数：关闭之后，也不进任何别人的材料（诊 2 本人、实现、提案都没有）", (t) => {
|
||||
const { l } = diagnosisStudy(t, "settled");
|
||||
l.append("stageOpen", { stageId: "s1-prop", stage: "proposal", round: 1, roster: ["a1", "a2"] });
|
||||
const proposals = [mat(l, "proposal", "a1", "s1-prop", null), mat(l, "proposal", "a2", "s1-prop", null)];
|
||||
for (const m of proposals) {
|
||||
for (const x of [...SEALED.a1, ...SEALED.a2]) lacks(m, x);
|
||||
}
|
||||
const own = mat(l, "diagnosisPrediction", "a1", "s1-dpred", null);
|
||||
for (const x of SEALED.a2) lacks(own, x);
|
||||
});
|
||||
|
||||
test("诊断实现：选中的实验原文、基线代码、原始产物的位置；不给信念本和成绩", (t) => {
|
||||
const { l, chosen } = diagnosisStudy(t, "selected");
|
||||
const m = mat(l, "diagnosisImplementation", "x1", "s1-dimpl", D);
|
||||
assert.deepEqual(m.checklist, { kind: "diagnosisImplementation", experiments: chosen });
|
||||
assert.deepEqual(m.readable, [{ version: "v-base", label: "起点代码（当前基线）", dir: "/code/v-base" }]);
|
||||
has(m, "产出当前基线的那次正式运行（run-0）的原始产物已经只读地放进你的工作目录");
|
||||
for (const id of chosen) has(m, `### ${id} `);
|
||||
assert.ok(!headings(m).includes("信念本") && !headings(m).includes("本批基线那次运行的结果"));
|
||||
lacks(m, "UNIQUE-A2"); // 诊断解释不给实验员：只给选中的实验
|
||||
});
|
||||
|
||||
test("诊断修复：实现的材料加退回意见（没通过的实验、依据、上一次位置表和改动）、能读上一次的诊断程序；通过的实验不要动", (t) => {
|
||||
const { l, chosen } = diagnosisStudy(t, "repairOpen");
|
||||
const m = mat(l, "diagnosisRepair", "x1", "s1-dimpl2", D, { changes: CHANGES_C1, artifacts: null });
|
||||
assert.deepEqual(m.checklist, { kind: "diagnosisRepair", experiments: chosen });
|
||||
assert.deepEqual(m.readable.map((x) => x.version), ["v-base", "v-diag"]);
|
||||
const opinion = m.sections.find((s) => s.heading === "退回意见")!.text;
|
||||
assert.ok(opinion.includes(`实验 ${chosen[1]}：没照写法做；solve.py:20`));
|
||||
assert.ok(!opinion.includes(`实验 ${chosen[0]}：`), "通过的实验不列");
|
||||
assert.ok(opinion.includes("核验员：第二个实验没照写法做"));
|
||||
assert.ok(opinion.includes(`"experiment": "${chosen[0]}"`) && opinion.includes("多起点循环"));
|
||||
});
|
||||
|
||||
test("诊断核验：选中的实验原文、基线到诊断程序的改动、位置表；不给信念本和实验员的说明", (t) => {
|
||||
const { l, chosen } = diagnosisStudy(t, "implemented");
|
||||
const m = mat(l, "diagnosisVerification", "v1", "s1-dver", D, { changes: CHANGES_C1, artifacts: null });
|
||||
assert.deepEqual(m.checklist, { kind: "diagnosisVerification", experiments: chosen });
|
||||
assert.deepEqual(m.readable.map((x) => x.version), ["v-base", "v-diag"]);
|
||||
has(m, "多起点循环");
|
||||
has(m, "solve.py:10-19");
|
||||
for (const id of chosen) has(m, `### ${id} `);
|
||||
assert.ok(!headings(m).includes("信念本"));
|
||||
lacks(m, "UNIQUE-X1");
|
||||
});
|
||||
|
||||
test("本轮诊断结果：提案材料里每个实验的做法、实测、用时、结局，每条解释确认或否定，推出的方向；排在基线之后、研究记录之前", (t) => {
|
||||
const { l, chosen } = diagnosisStudy(t, "settled");
|
||||
l.append("stageOpen", { stageId: "s1-prop", stage: "proposal", round: 1, roster: ["a1", "a2"] });
|
||||
const m = mat(l, "proposal", "a1", "s1-prop", null);
|
||||
const h = headings(m);
|
||||
assert.ok(h.indexOf("基线") < h.indexOf("本轮诊断结果") && h.indexOf("本轮诊断结果") < h.indexOf("研究记录"));
|
||||
const section = m.sections.find((s) => s.heading === "本轮诊断结果")!.text;
|
||||
const [first, second] = chosen as [string, string];
|
||||
assert.ok(section.includes(`诊断运行 run-1：在基线 base（最终指标 0.96）上，一次运行做了 1 个实验，只扣 1 次`));
|
||||
const line = (id: string) => section.split("\n").find((x) => x.startsWith(`- ${id} `))!;
|
||||
for (const part of ["实测 1.1，改进量 +0.14", "用时 31.5 秒", "其余指标：d1.gap 0.02", "测前合并范围"]) assert.ok(line(first!).includes(part), part);
|
||||
assert.ok(line(second!).includes("证据不足：核验不通过，第二个实验两次核验都没落实"));
|
||||
assert.match(section, /没做：一次诊断最多做 D 个实验，放不下/);
|
||||
assert.match(section, /- r1-a1-dx1（作者 a1）：确认 .*；否定 /);
|
||||
assert.ok(section.includes("甲的诊断解释：卡在起点 UNIQUE-A1") && section.includes("乙的诊断解释：卡在收尾 UNIQUE-A2"));
|
||||
assert.match(section, /程序按诊断推出的方向：d\d（实测 \+0\.14）。这只是按公式推出来的参考，你可以不跟。/);
|
||||
assert.ok(m.records.some((x) => x.id === "diagnosis-r1" && x.text.includes("run-1") && !x.text.includes("7.771")));
|
||||
assert.ok(m.readable.some((x) => x.version === "v-diag" && x.label === "第 1 轮诊断程序的代码"));
|
||||
});
|
||||
|
||||
/** 只有基线、第 1 轮提案阶段开着的账本：本轮没有诊断记录。 */
|
||||
function bare(t: TestContext): Ledger {
|
||||
const { ledger: l } = freshLedger(t);
|
||||
const w = new Writer(l);
|
||||
l.append("code", { object: "base", version: "v-base", locations: null });
|
||||
w.run("run-0", "baseline", 0, null, judged(0.96, "main"));
|
||||
w.open("s1-prop", "proposal", 1, ["a1", "a2"]);
|
||||
return l;
|
||||
}
|
||||
const diagnosisText = (m: TaskMaterial) => m.sections.find((s) => s.heading === "本轮诊断结果")!.text;
|
||||
|
||||
test("本轮没有诊断：写原因。余额不到 2 次、配置关掉（余额够却没有决定）", (t) => {
|
||||
const l = bare(t);
|
||||
// 基线占 1 次：运行总数 2 时余额 1（不到 2），总数 10 时余额 9。
|
||||
assert.equal(diagnosisText(mat(l, "proposal", "a1", "s1-prop", null, NONE, { ...study, totalRuns: 2 })), "本轮没有诊断：余额只剩 1 次，不到 2 次，要留 1 次给本轮正向");
|
||||
assert.equal(diagnosisText(mat(l, "proposal", "a1", "s1-prop", null)), "本轮没有诊断：配置关掉了诊断");
|
||||
});
|
||||
|
||||
test("本轮没有诊断：信息价值不抵成本时，原因带最好的组合净值；选中的实验一个都没剩下时，原因取事件", (t) => {
|
||||
const l = bare(t);
|
||||
const w = new Writer(l);
|
||||
const input = ruleInput(9, true); // 两条解释对每个实验预测一样：信息价值只剩开放预留那一点，抵不过成本
|
||||
const decision = selectDiagnosis(input);
|
||||
assert.equal(decision.diagnose, false);
|
||||
w.decision("selectDiagnosis", 1, input, decision);
|
||||
const best = Math.max(...decision.combos.map((c) => c.net));
|
||||
const text = diagnosisText(mat(l, "proposal", "a1", "s1-prop", null));
|
||||
assert.equal(text, `本轮没有诊断：信息价值减成本都不大于 0，最好的组合净值 ${Number(best.toPrecision(4))}`);
|
||||
|
||||
// 同一轮另一份账本：决定是做，但核验都没通过、修复用完。
|
||||
const l2 = bare(t);
|
||||
const w2 = new Writer(l2);
|
||||
const done = selectDiagnosis(ruleInput(9, false));
|
||||
w2.decision("selectDiagnosis", 1, ruleInput(9, false), done);
|
||||
l2.append("event", diagnosisNotRunEvent(1, "d1、d2 都没通过核验"));
|
||||
assert.equal(diagnosisText(mat(l2, "proposal", "a1", "s1-prop", null)), "本轮没有诊断：选中的实验核验都没通过、修复次数用完，一个都没跑：d1、d2 都没通过核验");
|
||||
});
|
||||
|
||||
test("诊断运行在诊断结算写入之前不公开，写入之后公开（和正向的采纳、消融的结算同一类规则）", (t) => {
|
||||
const run = (l: Ledger) => new Board(l.records()).runs().find((x) => x.plan.run === "run-1")!;
|
||||
const running = diagnosisStudy(t, "run").l;
|
||||
assert.equal(new Board(running.records()).runPublic(run(running)), false);
|
||||
const settled = diagnosisStudy(t, "settled").l;
|
||||
assert.equal(new Board(settled.records()).runPublic(run(settled)), true);
|
||||
});
|
||||
|
||||
test("信念本的诊断历史：各轮的实验、确认与否定、方向；战绩按人加一行诊断；诊断实验的实测不算手里最好", (t) => {
|
||||
const { l, chosen } = diagnosisStudy(t, "settled");
|
||||
l.append("stageOpen", { stageId: "s1-prop", stage: "proposal", round: 1, roster: ["a1", "a2"] });
|
||||
const book = mat(l, "proposal", "a1", "s1-prop", null).sections.find((s) => s.heading === "信念本")!.text;
|
||||
assert.ok(book.includes("**诊断历史**"));
|
||||
assert.ok(book.includes(`- 第 1 轮：\n 诊断运行 run-1：在基线 base`));
|
||||
assert.ok(book.includes(` - ${chosen[0]} `) && book.includes("确认 d1；否定 无"));
|
||||
assert.ok(book.includes("手里最好 0.96"), "诊断实验实测 1.1，但诊断的答案不采纳，手里最好仍是基线");
|
||||
// 战绩：a1、a2 各一条诊断：只有 d1 一个有效实验，样本各 1 个。a1 的 q5 = 1.25 > 实测 1.1 偏高；a2 的 q5 = 0.95 < 1.1 偏低。
|
||||
assert.ok(book.includes("- a1，诊断 1 次：中位数偏高 1、偏低 0"));
|
||||
assert.ok(book.includes("- a2，诊断 1 次：中位数偏高 0、偏低 1"));
|
||||
});
|
||||
|
||||
test("信念本的诊断历史：第 2 轮诊断关着时，第 1 轮的历史还在，第 2 轮没走完不列；第 2 轮诊断任务读不到第 2 轮的诊断结果", (t) => {
|
||||
const { l } = diagnosisStudy(t, "settled");
|
||||
l.append("stageOpen", { stageId: "s1-prop", stage: "proposal", round: 1, roster: ["a1", "a2"] });
|
||||
l.append("stageClose", { stageId: "s1-prop" });
|
||||
// 第 1 轮采纳 r1-c1 作基线要写一大串记录，这里只看第 2 轮的诊断任务：基线仍是 base，所以直接开第 2 轮的诊断阶段。
|
||||
l.append("stageOpen", { stageId: "s2-dp", stage: "diagnosisProposal", round: 2, roster: ["a1", "a2"] });
|
||||
const m = mat(l, "diagnosisProposal", "a1", "s2-dp", null, { changes: null, artifacts: {} });
|
||||
assert.ok(!headings(m).includes("本轮诊断结果"));
|
||||
const book = m.sections.find((s) => s.heading === "信念本")!.text;
|
||||
assert.ok(book.includes("- 第 1 轮：") && !book.includes("- 第 2 轮："));
|
||||
});
|
||||
|
||||
test("实现正向候选的实验员：本轮有诊断时能读诊断程序（只读，注明冻结协议引用诊断实验时作参考）；没有诊断时没有", (t) => {
|
||||
const { l } = diagnosisStudy(t, "settled");
|
||||
const w = new Writer(l);
|
||||
l.append("stageOpen", { stageId: "s1-prop", stage: "proposal", round: 1, roster: ["a1", "a2"] });
|
||||
w.protocol("r1-c1", protocolOf("r1-c1", P_A1, 1, "base", ["r1-a1-1"], ["加多起点"]));
|
||||
w.open("s1-impl-c1", "implementation", 1, ["x1"]);
|
||||
const m = mat(l, "implementation", "x1", "s1-impl-c1", "r1-c1");
|
||||
assert.deepEqual(m.readable, [
|
||||
{ version: "v-base", label: "起点代码（本批基线）", dir: "/code/v-base" },
|
||||
{ version: "v-diag", label: "本轮诊断程序（只读，冻结协议引用诊断实验时作参考）", dir: "/code/v-diag" },
|
||||
]);
|
||||
has(m, "冻结协议引用诊断实验时作参考");
|
||||
lacks(m, "UNIQUE-A1"); // 只给代码，不给诊断解释和成绩
|
||||
lacks(m, "实测 1.1");
|
||||
|
||||
const l2 = bare(t);
|
||||
const w2 = new Writer(l2);
|
||||
w2.protocol("r1-c1", protocolOf("r1-c1", P_A1, 1, "base", ["r1-a1-1"], ["加多起点"]));
|
||||
w2.open("s1-impl-c1", "implementation", 1, ["x1"]);
|
||||
assert.deepEqual(mat(l2, "implementation", "x1", "s1-impl-c1", "r1-c1").readable.map((x) => x.version), ["v-base"]);
|
||||
});
|
||||
@@ -36,6 +36,7 @@ export const study: StudyFacts = {
|
||||
lambda: 0.1,
|
||||
totalRuns: 10,
|
||||
proposalLimit: 2,
|
||||
diagnosisExperimentLimit: 2,
|
||||
factorLimit: 3,
|
||||
metrics: [
|
||||
{ id: "sum-of-radii", layer: "final", meaning: "半径和", unit: "无", direction: "maximize", howMeasured: "校验器", missing: "invalidateRun" },
|
||||
@@ -114,7 +115,7 @@ export class Writer {
|
||||
decision(decision: string, round: number, input: unknown, output: unknown): void {
|
||||
this.l.append("decision", { decision, round, rule: "test", ruleVersion: 1, params: null, seed: null, input: input as never, output: output as never });
|
||||
}
|
||||
run(run: string, purpose: "baseline" | "forward" | "ablation", round: number, object: string | null, j: JudgedResult): void {
|
||||
run(run: string, purpose: "baseline" | "forward" | "ablation" | "diagnosis", round: number, object: string | null, j: JudgedResult | null): void {
|
||||
this.l.append("runPlanned", { run, purpose, round, object, definition });
|
||||
this.l.append("runStarted", { run });
|
||||
this.l.append("runResult", { run, process: exited, judged: j });
|
||||
|
||||
@@ -0,0 +1,294 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { test } from "node:test";
|
||||
import {
|
||||
diagnosisModel,
|
||||
runDiagnosisRule,
|
||||
selectDiagnosis,
|
||||
settleDiagnosis,
|
||||
trackRecord,
|
||||
type DiagnosisExplanation,
|
||||
type DiagnosisRuleInput,
|
||||
type ExperimentOutcome,
|
||||
} from "../src/rules/index.ts";
|
||||
import { close, loadFixture, scenarioPicker, type Scenario } from "./rules-fixtures-helpers.ts";
|
||||
|
||||
// 预期值来自 tests/fixtures/rules/gen_diagnosis.py：只照方法.md 6.12、6.7、6.9 的文字、用有理数写成的独立参考，
|
||||
// 没有读过 src/rules/diagnosis.ts 和 value.ts（重新生成：python3 tests/fixtures/rules/gen_diagnosis.py）。
|
||||
// 场景里有手算的几个（opposite_lambda0 的 VOI = 0.1，same_lambda0 的 VOI = 0，说明写在场景的 note 里）。
|
||||
|
||||
const fixture = loadFixture("diagnosis.json");
|
||||
const pick = scenarioPicker(fixture);
|
||||
|
||||
const EXACT = 1e-12;
|
||||
const BITS = 1e-9;
|
||||
/** 只做一个实验时的 VOI：实现用网格近似，参考是逐段闭式积分。 */
|
||||
const closeK1 = (actual: number, expected: number, what: string): void =>
|
||||
close(actual, expected, 1e-9 + 2e-4 * Math.abs(expected), what);
|
||||
|
||||
type Ranges = Record<string, { q1: number; q5: number; q9: number }>;
|
||||
|
||||
function explanationsOf(s: Scenario): DiagnosisExplanation[] {
|
||||
return s.input.explanations.map((e: { id: string; author: string; ranges: Ranges }) => ({
|
||||
id: e.id,
|
||||
author: e.author,
|
||||
experiments: Object.entries(e.ranges).map(([experiment, range]) => ({ experiment, range })),
|
||||
}));
|
||||
}
|
||||
|
||||
function ruleInput(s: Scenario): DiagnosisRuleInput {
|
||||
return {
|
||||
baseline: s.input.b,
|
||||
direction: s.input.direction,
|
||||
lambda: s.input.lambda,
|
||||
experiments: s.input.experiments,
|
||||
explanations: explanationsOf(s),
|
||||
balance: s.input.balance,
|
||||
batchCap: s.input.batchCap,
|
||||
maxExperiments: s.input.maxExperiments,
|
||||
u: s.input.u,
|
||||
decay: s.input.decay,
|
||||
};
|
||||
}
|
||||
|
||||
/** 选实验场景：事前数字、组合表、选中的实验都和独立参考一致。 */
|
||||
function checkSelection(name: string): ReturnType<typeof selectDiagnosis> {
|
||||
const s = pick.get(name);
|
||||
const got = selectDiagnosis(ruleInput(s));
|
||||
const e = s.expected;
|
||||
for (const [d, v] of Object.entries(e.priorGains as Record<string, number>)) {
|
||||
close(got.priorGains.find((x) => x.experiment === d)!.gain, v, EXACT, `${name} ${d} 事前期望收益`);
|
||||
}
|
||||
close(got.g0, e.g0, EXACT, `${name} ĝ₀`);
|
||||
e.weights.forEach((w: number, j: number) => close(got.weights[j]!, w, EXACT, `${name} 起始权重 ${j}`));
|
||||
assert.deepEqual(got.combos.map((c) => c.experiments), e.combos.map((c: any) => c.experiments), `${name} 组合列表`);
|
||||
e.combos.forEach((c: any, i: number) => {
|
||||
const g = got.combos[i]!;
|
||||
if (c.voi === null) return;
|
||||
const what = `${name} {${c.experiments}}`;
|
||||
if (c.experiments.length === s.input.experiments.length) close(g.voi, c.voi, EXACT, `${what} VOI`);
|
||||
else closeK1(g.voi, c.voi, `${what} VOI`);
|
||||
assert.equal(g.rHat, c.rHat, `${what} R̂`);
|
||||
close(g.mPrime, c.mPrime, EXACT, `${what} m′`);
|
||||
close(g.cost, c.cost, EXACT, `${what} 成本`);
|
||||
close(g.value, c.value, 1e-9 + 2e-4 * Math.abs(c.value), `${what} 价值`);
|
||||
close(g.net, c.net, 1e-9 + 2e-4 * Math.abs(c.value), `${what} 净值`);
|
||||
});
|
||||
if ("diagnose" in e) {
|
||||
assert.equal(got.diagnose, e.diagnose, `${name} 做不做`);
|
||||
assert.deepEqual(got.chosen, e.chosen, `${name} 选中的实验`);
|
||||
}
|
||||
return got;
|
||||
}
|
||||
|
||||
test("6.12 信息价值：两条解释对实验预测相反时为正（手算 0.1），预测一样时为 0", () => {
|
||||
const opposite = checkSelection("opposite_lambda0");
|
||||
close(opposite.combos[0]!.voi, 0.1, EXACT, "手算：0.5 × 0.2 − 0");
|
||||
assert.equal(opposite.diagnose, true);
|
||||
|
||||
const same = checkSelection("same_lambda0");
|
||||
close(same.combos[0]!.voi, 0, EXACT, "预测一样、范围都在 0 以上：做不做都不改变看法");
|
||||
assert.equal(same.diagnose, false, "净值 = −成本 < 0");
|
||||
assert.equal(same.experiments[0]!.reason, "value");
|
||||
assert.equal(same.chosenNet, 0);
|
||||
|
||||
// λ = 0.1：预测一样时只剩开放预留带来的一小部分，比预测相反小得多。
|
||||
const oppositeReserve = checkSelection("opposite_lambda01");
|
||||
const sameReserve = checkSelection("same_lambda01");
|
||||
close(sameReserve.combos[0]!.voi, 0.0025, EXACT, "手算：λ × (E[max(0, U)] − E U) = 0.1 × 0.025");
|
||||
assert.ok(sameReserve.combos[0]!.voi > 0);
|
||||
assert.ok(oppositeReserve.combos[0]!.voi > 10 * sameReserve.combos[0]!.voi);
|
||||
});
|
||||
|
||||
test("6.12 价值减成本：不管做几个实验成本相同；R̂ = ⌈(余额 − 1) ÷ 每批上限⌉", () => {
|
||||
const got = checkSelection("multi_all");
|
||||
// 手算：余额 9、K = 2：R̂ = ⌈8 ÷ 2⌉ = 4，m′ = 1 + 0.5 + 0.25 + 0.125 = 1.875；
|
||||
// 成本 = 2 × 0.0525 × (3/4 − 2/3) = 0.00875（ĝ₀ = 0.0525 来自参考脚本的事前期望）。
|
||||
assert.equal(got.combos.length, 1, "实验数 3 ≤ D = 4：只有全做一个组合");
|
||||
assert.equal(got.combos[0]!.rHat, 4);
|
||||
close(got.combos[0]!.mPrime, 1.875, EXACT, "m′");
|
||||
close(got.combos[0]!.cost, 0.00875, EXACT, "成本");
|
||||
assert.deepEqual(got.chosen, ["d1", "d2", "d3"]);
|
||||
assert.ok(got.experiments.every((x) => x.done && x.reason === null && x.bestCombo === null));
|
||||
});
|
||||
|
||||
test("6.12 选实验：余额 < 2 时不诊断，没有可选组合", () => {
|
||||
const got = checkSelection("balance_one");
|
||||
assert.deepEqual(got.combos, []);
|
||||
assert.equal(got.diagnose, false);
|
||||
assert.deepEqual(got.chosen, []);
|
||||
assert.ok(got.experiments.every((x) => !x.done && x.bestCombo === null));
|
||||
});
|
||||
|
||||
test("6.12 选实验：实验数超过 D 时只比较装满的组合", () => {
|
||||
const one = checkSelection("over_cap_d1");
|
||||
assert.equal(one.combos.length, 3, "C(3,1) 个单实验组合");
|
||||
assert.ok(one.combos.every((c) => c.experiments.length === 1), "没有空组合，也没有更大的组合");
|
||||
assert.deepEqual(one.chosen, ["d2"]);
|
||||
// 没选中的实验：放不下，并附上含它的组合里净值最大的那个（这里就是它自己的单实验组合）。
|
||||
const d3 = one.experiments.find((x) => x.experiment === "d3")!;
|
||||
assert.equal(d3.done, false);
|
||||
assert.equal(d3.reason, "budget");
|
||||
assert.deepEqual(d3.bestCombo!.experiments, ["d3"]);
|
||||
close(d3.bestCombo!.net, pick.get("over_cap_d1").expected.combos[2].net, 1e-9 + 2e-4 * 0.1, "d3 的净值");
|
||||
|
||||
const two = checkSelection("over_cap_d2");
|
||||
assert.deepEqual(two.combos.map((c) => c.experiments), [["d1", "d2"], ["d1", "d3"], ["d2", "d3"]]);
|
||||
assert.equal(two.chosen.length, 2);
|
||||
// 多做一个实验信息价值只会不变或变大：每个两实验组合的 VOI 不小于它包含的任一单实验（参考脚本的精确值）。
|
||||
const singles = pick.get("over_cap_d1").expected.combos as Array<{ experiments: string[]; voi: number }>;
|
||||
for (const pair of two.combos) {
|
||||
for (const d of pair.experiments) {
|
||||
const single = singles.find((c) => c.experiments[0] === d)!.voi;
|
||||
assert.ok(pair.voi >= single - (1e-9 + 2e-4 * single), `{${pair.experiments}} 的 VOI ${pair.voi} 小于 {${d}} 的 ${single}`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test("6.12 选实验：平票按实验编号（自然序，d2 排在 d10 前面）；越小越好按改进量算", () => {
|
||||
const tie = checkSelection("tie_by_id");
|
||||
assert.deepEqual(tie.chosen, ["d2"]);
|
||||
assert.deepEqual(tie.experiments.find((x) => x.experiment === "d10")!.bestCombo!.experiments, ["d10"]);
|
||||
checkSelection("minimize_all");
|
||||
});
|
||||
|
||||
test("登记表：选诊断实验只有一条规则，结果带规则名和版本；同样的输入算两次逐位相同", () => {
|
||||
const s = pick.get("multi_all");
|
||||
const run = runDiagnosisRule("valueMinusCost", ruleInput(s));
|
||||
assert.deepEqual(run.rule, { name: "valueMinusCost", version: 1 });
|
||||
assert.deepEqual(JSON.parse(JSON.stringify(run)), JSON.parse(JSON.stringify({ rule: run.rule, ...selectDiagnosis(ruleInput(s)) })));
|
||||
});
|
||||
|
||||
test("成分模型：每条解释一个成分加开放预留；解释合起来 1 − λ，同一人平分", () => {
|
||||
const s = pick.get("settle_mixed");
|
||||
const m = diagnosisModel({ ...ruleInput({ ...s, input: { ...s.input, balance: 9, batchCap: 2, maxExperiments: 4, u: 0.5, decay: 0.5 } }), });
|
||||
// 手算：researcher-1 一条 0.9 × 1/2，researcher-2 两条各 0.9 × 1/4，预留 0.1。
|
||||
assert.deepEqual(m.components, ["e1", "e2", "e3", "openReserve"]);
|
||||
[0.45, 0.225, 0.225, 0.1].forEach((w, j) => close(m.weights[j]!, w, EXACT, `权重 ${j}`));
|
||||
});
|
||||
|
||||
type Settled = ReturnType<typeof settleDiagnosis>;
|
||||
|
||||
function settle(name: string): { got: Settled; e: any } {
|
||||
const s = pick.get(name);
|
||||
const outcomes = s.input.outcomes as Record<string, { kind: ExperimentOutcome["kind"]; score?: number }>;
|
||||
const got = settleDiagnosis({
|
||||
baseline: s.input.b,
|
||||
direction: s.input.direction,
|
||||
epsilon: s.input.epsilon,
|
||||
lambda: s.input.lambda,
|
||||
experiments: Object.entries(outcomes).map(([experiment, o]) => ({
|
||||
experiment,
|
||||
outcome: o.kind === "valid" ? { kind: "valid", score: o.score! } : ({ kind: o.kind } as ExperimentOutcome),
|
||||
})),
|
||||
explanations: explanationsOf(s),
|
||||
});
|
||||
const e = s.expected;
|
||||
assert.deepEqual(got.experiments.map((x) => x.outcome), Object.keys(e.experiments).map((d) => e.experiments[d].outcome), `${name} 结局`);
|
||||
for (const row of got.experiments) {
|
||||
const x = e.experiments[row.experiment];
|
||||
const what = `${name} ${row.experiment}`;
|
||||
close(row.merged.q1, x.merged.q1, BITS, `${what} 合并一成`);
|
||||
close(row.merged.q5, x.merged.q5, BITS, `${what} 合并一半`);
|
||||
close(row.merged.q9, x.merged.q9, BITS, `${what} 合并九成`);
|
||||
if (x.gain === undefined) {
|
||||
// 证据不足、没做：不判、不记账。
|
||||
assert.equal(row.gain, null, `${what} 收益`);
|
||||
assert.equal(row.worldBits, null, `${what} 世界比特`);
|
||||
assert.equal(row.surprise, null, `${what} 惊讶`);
|
||||
assert.equal(row.disagreement, null, `${what} 分歧量`);
|
||||
} else {
|
||||
close(row.gain!, x.gain, EXACT, `${what} 收益`);
|
||||
close(row.worldBits!, x.worldBits, BITS, `${what} 世界比特`);
|
||||
assert.equal(row.surprise!.cell, x.surprise.bin, `${what} 惊讶所在格`);
|
||||
close(row.surprise!.prior, x.surprise.p, BITS, `${what} 事前把握`);
|
||||
assert.equal(row.surprise!.bits.status, x.surprise.status === "ok" ? "measured" : "unmeasured");
|
||||
if (row.surprise!.bits.status === "measured") close(row.surprise!.bits.value, x.surprise.surprise, BITS, `${what} 惊讶`);
|
||||
close(row.disagreement!, x.disagreement, BITS, `${what} 分歧量`);
|
||||
}
|
||||
}
|
||||
// 确认与否定：只列有效实验。
|
||||
assert.deepEqual(
|
||||
Object.fromEntries(got.judgments.map((j) => [j.id, Object.fromEntries(j.verdicts.map((v) => [v.experiment, v.verdict]))])),
|
||||
e.judgments,
|
||||
`${name} 确认与否定`,
|
||||
);
|
||||
const dir = got.direction;
|
||||
assert.equal(dir.weightsKept, e.direction.weightsKept, `${name} 权重是否保持`);
|
||||
e.direction.weights.forEach((w: number, j: number) => close(dir.weights[j]!, w, EXACT, `${name} 新权重 ${j}`));
|
||||
for (const g of dir.gains) {
|
||||
const x = e.direction.gains[g.experiment];
|
||||
close(g.gain, x.gain, EXACT, `${name} ${g.experiment} 方向收益`);
|
||||
assert.equal(g.measured, x.measured, `${name} ${g.experiment} 是实测还是估计`);
|
||||
}
|
||||
assert.equal(dir.experiment, e.direction.experiment, `${name} 推出的方向`);
|
||||
close(dir.gain, e.direction.gain, EXACT, `${name} 方向收益`);
|
||||
assert.equal(dir.found, e.direction.found, `${name} 有没有指出方向`);
|
||||
return { got, e };
|
||||
}
|
||||
|
||||
test("6.12 结算：证据不足的实验不判、不记账；没做的实验也不判", () => {
|
||||
const { got } = settle("settle_mixed");
|
||||
assert.deepEqual(got.experiments.map((x) => x.outcome), ["valid", "insufficient", "untested"]);
|
||||
// d2、d3 没有有效分数：每条解释只有 d1 一行判定，没有比特。
|
||||
assert.ok(got.judgments.every((j) => j.verdicts.length === 1 && j.verdicts[0]!.experiment === "d1"));
|
||||
assert.equal(got.experiments.filter((x) => x.worldBits !== null).length, 1);
|
||||
// 手算：d1 实测 1.045 = b + 0.045，收益 +0.045；落在 e1 的 [1.02, 1.08] 里确认，e2 的 [1.00, 1.06] 里确认，e3 的 [0.98, 1.10] 里确认。
|
||||
assert.deepEqual(got.judgments.map((j) => j.verdicts[0]!.verdict), ["confirmed", "confirmed", "confirmed"]);
|
||||
});
|
||||
|
||||
test("6.12 结算：两个有效实验、确认与否定、权重更新", () => {
|
||||
const { got } = settle("settle_two_valid");
|
||||
// d2 实测 1.25，e2 的 [1.1, 1.3] 确认；e1 的 [0.95, 1.05]、e3 的 [0.8, 1.0] 否定。
|
||||
assert.deepEqual(got.judgments.find((j) => j.id === "e2")!.verdicts.find((v) => v.experiment === "d2")!.verdict, "confirmed");
|
||||
assert.deepEqual(got.judgments.find((j) => j.id === "e1")!.verdicts.find((v) => v.experiment === "d2")!.verdict, "denied");
|
||||
assert.deepEqual(got.judgments.find((j) => j.id === "e3")!.verdicts.find((v) => v.experiment === "d2")!.verdict, "denied");
|
||||
});
|
||||
|
||||
test("6.12 结算：全部实验收益不为正时，记没有指出能涨分的方向", () => {
|
||||
const { got } = settle("settle_no_direction");
|
||||
assert.equal(got.direction.found, false);
|
||||
assert.ok(got.direction.gains.every((g) => g.gain <= 0));
|
||||
// 手算：收益分别是 0.97 − 1 = −0.03、0.8 − 1 = −0.2、0.9 − 1 = −0.1，最大的是 d1 的 −0.03。
|
||||
assert.equal(got.direction.experiment, "d1");
|
||||
close(got.direction.gain, -0.03, EXACT, "最大收益");
|
||||
});
|
||||
|
||||
test("6.12 结算：没有有效实验、实测落在所有成分之外时保持起始权重", () => {
|
||||
const none = settle("settle_all_insufficient");
|
||||
assert.equal(none.got.direction.weightsKept, false, "没有有效实验时没有密度可乘，权重原样（不算'无从更新'）");
|
||||
assert.ok(none.got.judgments.every((j) => j.verdicts.length === 0));
|
||||
const outside = settle("settle_outside_all");
|
||||
assert.equal(outside.got.direction.weightsKept, true);
|
||||
assert.deepEqual(outside.got.direction.weights, diagnosisModel({
|
||||
baseline: 1, direction: "maximize", lambda: 0.1, experiments: ["d1", "d2", "d3"],
|
||||
explanations: explanationsOf(pick.get("settle_outside_all")),
|
||||
}).weights);
|
||||
});
|
||||
|
||||
test("6.12 结算：越小越好按改进量 b − x", () => {
|
||||
const { got } = settle("settle_minimize");
|
||||
close(got.experiments[0]!.gain!, 0.12, EXACT, "手算：1.0 − 0.88");
|
||||
close(got.experiments[1]!.gain!, -0.05, EXACT, "手算：1.0 − 1.05");
|
||||
assert.equal(got.direction.experiment, "d1");
|
||||
});
|
||||
|
||||
test("战绩：按人、按正向 / 诊断 / 消融分开数，次序是编号、正向、诊断、消融", () => {
|
||||
// 手算：researcher-2 的诊断样本 (q1, q5, q9) = (0.1, 0.2, 0.3)，实测 0.35 在 q9 之上：中位数偏低、被穿过上沿、没包住、落点 aboveQ9。
|
||||
const range = { q1: 0.1, q5: 0.2, q9: 0.3 };
|
||||
const rows = trackRecord([
|
||||
{ researcher: "researcher-2", phase: "ablation", range, value: 0.2 },
|
||||
{ researcher: "researcher-2", phase: "diagnosis", range, value: 0.35 },
|
||||
{ researcher: "researcher-1", phase: "diagnosis", range, value: 0.05 },
|
||||
{ researcher: "researcher-2", phase: "forward", range, value: 0.25 },
|
||||
]);
|
||||
assert.deepEqual(rows.map((r) => `${r.researcher}/${r.phase}`), [
|
||||
"researcher-1/diagnosis", "researcher-2/forward", "researcher-2/diagnosis", "researcher-2/ablation",
|
||||
]);
|
||||
const t = rows[2]!.tally;
|
||||
assert.deepEqual([t.total, t.medianLow, t.medianHigh, t.aboveTop, t.belowBottom, t.covered], [1, 1, 0, 1, 0, 0]);
|
||||
assert.equal(t.placements.aboveQ9, 1);
|
||||
});
|
||||
|
||||
test("diagnosis.json 里的场景都被测到", () => {
|
||||
assert.deepEqual(pick.unused(), []);
|
||||
});
|
||||
@@ -2,6 +2,7 @@ import assert from "node:assert/strict";
|
||||
import { test, type TestContext } from "node:test";
|
||||
import {
|
||||
ablationOutcome,
|
||||
diagnosisOutcome,
|
||||
protocolBody,
|
||||
receive,
|
||||
verificationOutcome,
|
||||
@@ -442,3 +443,138 @@ test("消融实现的代码存不成版本：退回", async () => {
|
||||
});
|
||||
assert.match(reason, /代码存不成版本：bin\/x（符号链接/);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 诊断（方法.md 4.3 诊 1–诊 4）
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const DX_EXPERIMENT = (name: string, missing: "recordMissing" | "invalidateRun" = "recordMissing", ids: string[] = []) => ({
|
||||
name,
|
||||
spot: "solve.py 的起点生成",
|
||||
body: "换成更强的起点，交出答案",
|
||||
metrics: ids.map((id) => ({ ...metric(id), missing })),
|
||||
});
|
||||
const DX_PROPOSAL = (over: Partial<Submission<"diagnosisProposal">> = {}): Submission<"diagnosisProposal"> => ({
|
||||
explanations: [{ body: "卡在起点" }, { body: "卡在收尾" }],
|
||||
experiments: [DX_EXPERIMENT("强起点"), DX_EXPERIMENT("拿掉收尾", "recordMissing", ["raw-score"])],
|
||||
...over,
|
||||
});
|
||||
const DX_LIST: ChecklistOf<"diagnosisProposal"> = { kind: "diagnosisProposal", maxExplanations: 2, maxExperiments: 2, takenMetricIds: ["sum-of-radii", "answer-path"] };
|
||||
const dxProposal = (content: Submission<"diagnosisProposal">): Incoming => ({ kind: "diagnosisProposal", content, checklist: DX_LIST, known: null, jevModel: MODEL });
|
||||
|
||||
test("诊断解释与实验：解释 1 至 M 条、实验 1 至 A 个；超过就退回", async () => {
|
||||
const many = DX_PROPOSAL({ explanations: [{ body: "a" }, { body: "b" }, { body: "c" }], experiments: [DX_EXPERIMENT("1"), DX_EXPERIMENT("2"), DX_EXPERIMENT("3")] });
|
||||
const reason = await rejected(dxProposal(many));
|
||||
assert.match(reason, /交了 3 条诊断解释，最多 2 条/);
|
||||
assert.match(reason, /交了 3 个诊断实验，最多 2 个/);
|
||||
});
|
||||
|
||||
test("诊断解释与实验：登记的指标只能是 recordMissing；同一个实验里不重名、不占用已有编号", async () => {
|
||||
const reason = await rejected(
|
||||
dxProposal(
|
||||
DX_PROPOSAL({
|
||||
experiments: [
|
||||
DX_EXPERIMENT("甲", "invalidateRun", ["raw-score"]),
|
||||
{ ...DX_EXPERIMENT("乙"), metrics: [metric("a"), metric("a"), metric("answer-path")] },
|
||||
],
|
||||
}),
|
||||
),
|
||||
);
|
||||
assert.match(reason, /第 1 个实验的指标 raw-score 选了 invalidateRun：诊断实验里的指标只能选 recordMissing/);
|
||||
assert.match(reason, /第 2 个实验的 metrics 里编号 a 写了不止一次/);
|
||||
assert.match(reason, /第 2 个实验的指标编号 answer-path 已被占用/);
|
||||
});
|
||||
|
||||
test("诊断解释与实验：对每个实验问冻结三问；两个实验用同一个指标短名没问题（冻结时各带自己实验的前缀）", async (t) => {
|
||||
fakeJev(t, () => false);
|
||||
const v = await receive(dxProposal(DX_PROPOSAL({ experiments: [DX_EXPERIMENT("甲", "recordMissing", ["x"]), DX_EXPERIMENT("乙", "recordMissing", ["x"])] })));
|
||||
assert.equal(v.accepted, true);
|
||||
assert.equal(v.jev!.length, 6, "两个实验各三问");
|
||||
assert.deepEqual(v.jev!.map((j) => j.question), ["self_contradictory", "changes_measurement", "too_vague", "self_contradictory", "changes_measurement", "too_vague"]);
|
||||
});
|
||||
|
||||
test("诊断解释与实验：Jev 判是就退回，原因写明第几个实验和哪一问", async (t) => {
|
||||
fakeJev(t, (q) => q === "too_vague");
|
||||
const v = await receive(dxProposal(DX_PROPOSAL()));
|
||||
assert.equal(v.accepted, false);
|
||||
const reason = (v as Extract<Verdict, { accepted: false }>).reason;
|
||||
assert.match(reason, /Jev 判出下面几处有问题/);
|
||||
assert.match(reason, /- 第 1 个诊断实验（强起点）：too_vague 概率 0\.99/);
|
||||
assert.match(reason, /- 第 2 个诊断实验（拿掉收尾）：too_vague 概率 0\.99/);
|
||||
});
|
||||
|
||||
const DX_PRED_LIST: ChecklistOf<"diagnosisPrediction"> = { kind: "diagnosisPrediction", explanations: ["r1-dx1", "r1-dx2"], experiments: ["d1", "d2"] };
|
||||
const dxPred = (predictions: Array<[string, string, ReturnType<typeof r>]>): Incoming => ({
|
||||
kind: "diagnosisPrediction",
|
||||
content: { predictions: predictions.map(([explanation, experiment, range]) => ({ explanation, experiment, range })), body: "理由" },
|
||||
checklist: DX_PRED_LIST,
|
||||
known: null,
|
||||
jevModel: MODEL,
|
||||
});
|
||||
|
||||
test("诊断预测：自己每条解释 × 每个合并后的实验，不多不少；收下时不问 Jev", async () => {
|
||||
const all: Array<[string, string, ReturnType<typeof r>]> = [
|
||||
["r1-dx1", "d1", r(1, 1.1, 1.2)], ["r1-dx1", "d2", r(1, 1.1, 1.2)], ["r1-dx2", "d1", r(1, 1.1, 1.2)], ["r1-dx2", "d2", r(1, 1.1, 1.2)],
|
||||
];
|
||||
assert.deepEqual(await receive(dxPred(all)), { accepted: true, jev: null, protocol: null });
|
||||
const reason = await rejected(dxPred([all[0]!, all[0]!, all[1]!, ["r1-dx3", "d1", r(1, 1.1, 1.2)]]));
|
||||
assert.match(reason, /predictions 的（解释:实验）应当是 r1-dx1:d1、r1-dx1:d2、r1-dx2:d1、r1-dx2:d2/);
|
||||
assert.match(reason, /你写的 r1-dx3:d1 不在里面/);
|
||||
assert.match(reason, /还缺 r1-dx2:d1、r1-dx2:d2/);
|
||||
assert.match(reason, /r1-dx1:d1 写了不止一次/);
|
||||
});
|
||||
|
||||
test("诊断预测：三个数要严格递增", async () => {
|
||||
const reason = await rejected(dxPred([["r1-dx1", "d1", r(1, 1, 1.2)], ["r1-dx1", "d2", r(1, 1.1, 1.2)], ["r1-dx2", "d1", r(1, 1.1, 1.2)], ["r1-dx2", "d2", r(1.3, 1.2, 1.1)]]));
|
||||
assert.match(reason, /r1-dx1:d1 的 range 的三个数要严格递增 q1 < q5 < q9，你写的是 1 \/ 1 \/ 1\.2/);
|
||||
assert.match(reason, /r1-dx2:d2 的 range 的三个数要严格递增/);
|
||||
});
|
||||
|
||||
const DX_CODE_BODY = { locations: [{ experiment: "d1", locations: ["solve.py:3-9"] }], concerns: "无", body: "无" };
|
||||
type DxCode = Extract<Incoming, { kind: "diagnosisImplementation" }>["known"]["code"];
|
||||
const dxCode = (kind: "diagnosisImplementation" | "diagnosisRepair", code: DxCode): Incoming =>
|
||||
kind === "diagnosisImplementation"
|
||||
? { kind, content: DX_CODE_BODY, checklist: { kind, experiments: ["d1"] }, known: { code }, jevModel: MODEL }
|
||||
: { kind, content: DX_CODE_BODY, checklist: { kind, experiments: ["d1"] }, known: { code }, jevModel: MODEL };
|
||||
|
||||
test("诊断实现、修复：改了不许改的文件、代码存不了就退回；有差异块时每块问一次五问", async (t) => {
|
||||
for (const kind of ["diagnosisImplementation", "diagnosisRepair"] as const) {
|
||||
assert.match(await rejected(dxCode(kind, { ...SAVED, protectedFiles: ["validate.py"] })), /改了不许改的文件：validate\.py/);
|
||||
assert.match(await rejected(dxCode(kind, { saved: false, path: "bin/x", reason: "符号链接" })), /代码存不成版本：bin\/x（符号链接）/);
|
||||
}
|
||||
fakeJev(t, () => false);
|
||||
const hunks = [
|
||||
{ path: "solve.py", change: "modified" as const, text: "@@ -1 +1 @@\n-a\n+b" },
|
||||
{ path: "solve.py", change: "modified" as const, text: "@@ -9 +9 @@\n-c\n+d" },
|
||||
];
|
||||
const v = await receive(dxCode("diagnosisRepair", { ...SAVED, hunks }));
|
||||
assert.equal(v.accepted, true);
|
||||
assert.equal(v.jev!.length, 10, "两块各五问");
|
||||
});
|
||||
|
||||
test("诊断核验：逐个实验的结果覆盖全部选中的实验", async () => {
|
||||
const verification = (ids: string[]): Incoming => ({
|
||||
kind: "diagnosisVerification",
|
||||
content: { experiments: ids.map((experiment) => ({ experiment, followed: true, nothingElse: true, where: "solve.py:3" })), body: "无" },
|
||||
checklist: { kind: "diagnosisVerification", experiments: ["d1", "d2"] },
|
||||
known: null,
|
||||
jevModel: MODEL,
|
||||
});
|
||||
assert.deepEqual(await receive(verification(["d2", "d1"])), { accepted: true, jev: null, protocol: null });
|
||||
const reason = await rejected(verification(["d1", "d3"]));
|
||||
assert.match(reason, /experiments 的实验应当是 d1、d2；你写的 d3 不在里面；还缺 d2/);
|
||||
});
|
||||
|
||||
test("诊断核验通过没有：逐个实验判，一个不通过不影响别的实验", () => {
|
||||
const v: Submission<"diagnosisVerification"> = {
|
||||
experiments: [
|
||||
{ experiment: "d1", followed: true, nothingElse: true, where: "a" },
|
||||
{ experiment: "d2", followed: false, nothingElse: true, where: "没找到" },
|
||||
{ experiment: "d3", followed: true, nothingElse: false, where: "顺手改了别处" },
|
||||
{ experiment: "d4", followed: false, nothingElse: false, where: "都不对" },
|
||||
],
|
||||
body: "无",
|
||||
};
|
||||
assert.deepEqual(diagnosisOutcome(v), { passed: false, failing: ["d2", "d3", "d4"], notFollowed: ["d2", "d4"], touchedElse: ["d3", "d4"] });
|
||||
assert.deepEqual(diagnosisOutcome({ experiments: [v.experiments[0]!], body: "无" }), { passed: true, failing: [], notFollowed: [], touchedElse: [] });
|
||||
});
|
||||
|
||||
@@ -0,0 +1,254 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { after, test, type TestContext } from "node:test";
|
||||
import { Runner } from "../src/runner/index.ts";
|
||||
import {
|
||||
diagnosisRun,
|
||||
diagnosisUnversionedNames,
|
||||
judgeDiagnosis,
|
||||
loadTask,
|
||||
placeBaselineArtifact,
|
||||
prepareDiagnosisJudging,
|
||||
prepareDiagnosisRun,
|
||||
type DiagnosisExperiment,
|
||||
type DiagnosisJudged,
|
||||
} from "../src/tasks/index.ts";
|
||||
import type { HypothesisMetric, Measured, ProcessResult, RunDefinition } from "../src/vocab/index.ts";
|
||||
|
||||
// 诊断运行的测试（任务书 18 第 1.4 节）：一次运行跑几个实验，经真实 Docker 在任务镜像里跑；没有 Docker 时失败，不跳过。
|
||||
// 镜像和容器用 inforsi-test-tasks-diag 开头的固定名字，与别的测试文件错开。
|
||||
// 判分的预期值是手算的 2 的幂有理数布局（同 tests/tasks.test.ts）：宽松布局半径和 26/128 = 0.203125，
|
||||
// 相切布局 26/64 = 0.40625。
|
||||
|
||||
const TASK_DIR = join(import.meta.dirname, "..", "tasks", "circle_packing");
|
||||
const IMAGE_TAG = "inforsi-test-tasks-diag";
|
||||
/** 测试用的总时限：6 个实验各分到 4 秒。 */
|
||||
const TOTAL_SECONDS = 24;
|
||||
|
||||
/** 任务目录抄一份，只把总时限改小，其余（Dockerfile、校验器、harness）原样，镜像编号也就相同。 */
|
||||
function shortTask() {
|
||||
const dir = mkdtempSync(join(tmpdir(), "inforsi-tasks-diag-task-"));
|
||||
cpSync(TASK_DIR, dir, { recursive: true });
|
||||
const facts = JSON.parse(readFileSync(join(dir, "task.json"), "utf8"));
|
||||
facts.run.wallSeconds = TOTAL_SECONDS;
|
||||
writeFileSync(join(dir, "task.json"), JSON.stringify(facts));
|
||||
return loadTask(dir);
|
||||
}
|
||||
|
||||
const task = shortTask();
|
||||
const runner = new Runner(null);
|
||||
const digest = await runner.buildImage(task.dir, IMAGE_TAG);
|
||||
after(() => {
|
||||
rmSync(task.dir, { recursive: true, force: true });
|
||||
docker(["rmi", IMAGE_TAG]);
|
||||
});
|
||||
|
||||
function docker(args: string[]): { status: number | null; stderr: string } {
|
||||
return spawnSync("docker", args, { encoding: "utf8" });
|
||||
}
|
||||
|
||||
function forceRemove(name: string): void {
|
||||
const removed = docker(["rm", "-f", name]);
|
||||
if (removed.status !== 0 && !removed.stderr.includes("No such container")) throw new Error(removed.stderr);
|
||||
}
|
||||
|
||||
function tempDir(t: TestContext, label: string): string {
|
||||
const dir = mkdtempSync(join(tmpdir(), `inforsi-tasks-diag-${label}-`));
|
||||
t.after(() => rmSync(dir, { recursive: true, force: true }));
|
||||
return dir;
|
||||
}
|
||||
|
||||
async function run(t: TestContext, suffix: string, definition: RunDefinition, dir: string): Promise<ProcessResult> {
|
||||
const name = `inforsi-test-tasks-diag-${suffix}`;
|
||||
forceRemove(name);
|
||||
t.after(() => forceRemove(name));
|
||||
const outcome = await runner.start(name, definition, dir);
|
||||
assert.deepEqual(outcome, { started: true });
|
||||
return runner.wait(name, tempDir(t, "log"));
|
||||
}
|
||||
|
||||
const m = (value: number): Measured => ({ status: "measured", value });
|
||||
|
||||
function rowLayout(radius: number, spacing: number, firstX: number): object {
|
||||
const n = 26;
|
||||
return {
|
||||
centers: Array.from({ length: n }, (_, i) => [firstX + i * spacing, 0.5]),
|
||||
radii: Array(n).fill(radius),
|
||||
reported_sum: 99,
|
||||
};
|
||||
}
|
||||
|
||||
const metric = (id: string): HypothesisMetric => ({ id, meaning: "测试用", unit: "无", direction: "maximize", howMeasured: "测试用", missing: "recordMissing" });
|
||||
|
||||
/** 六个实验：分别测"拿到了原始产物""指标不串""崩了不影响后面""超时""原始产物只读""答案不合法"。 */
|
||||
const EXPERIMENTS: DiagnosisExperiment[] = [
|
||||
{ id: "d-loose", metrics: [metric("d-loose.note")] },
|
||||
{ id: "d-same", metrics: [metric("d-same.note")] },
|
||||
{ id: "d-crash", metrics: [] },
|
||||
{ id: "d-slow", metrics: [] },
|
||||
{ id: "d-tamper", metrics: [] },
|
||||
{ id: "d-outside", metrics: [] },
|
||||
];
|
||||
|
||||
const PROGRAM = `import json
|
||||
import time
|
||||
|
||||
|
||||
def layout(radius, spacing, first_x):
|
||||
return ([[first_x + i * spacing, 0.5] for i in range(26)], [radius] * 26, 99.0)
|
||||
|
||||
|
||||
def run_experiment(experiment_id, baseline_dir):
|
||||
if experiment_id == "d-loose":
|
||||
with open("metrics.json", "w") as handle:
|
||||
json.dump({"d-loose.note": 0.5}, handle)
|
||||
return layout(1 / 128, 1 / 32, 7 / 64)
|
||||
if experiment_id == "d-same":
|
||||
with open(baseline_dir + "/solution.json") as handle:
|
||||
solution = json.load(handle)
|
||||
return solution["centers"], solution["radii"], solution["reported_sum"]
|
||||
if experiment_id == "d-crash":
|
||||
raise RuntimeError("故意崩")
|
||||
if experiment_id == "d-slow":
|
||||
time.sleep(60)
|
||||
if experiment_id == "d-tamper":
|
||||
with open(baseline_dir + "/solution.json", "w") as handle:
|
||||
handle.write("{}")
|
||||
if experiment_id == "d-outside":
|
||||
return layout(1 / 128, 1 / 32, 2.0)
|
||||
raise RuntimeError("不认识的实验 " + experiment_id)
|
||||
`;
|
||||
|
||||
/** 基线那次正式运行的运行目录：只要有 solution.json。放的是相切布局，半径和 0.40625。 */
|
||||
function baselineRunDir(t: TestContext): string {
|
||||
const dir = tempDir(t, "baseline-run");
|
||||
writeFileSync(join(dir, "solution.json"), JSON.stringify(rowLayout(1 / 64, 1 / 32, 7 / 64)));
|
||||
return dir;
|
||||
}
|
||||
|
||||
function programDir(t: TestContext): string {
|
||||
const dir = tempDir(t, "program");
|
||||
writeFileSync(join(dir, "initial.py"), PROGRAM);
|
||||
return dir;
|
||||
}
|
||||
|
||||
function value(item: Measured | undefined): number {
|
||||
assert.equal(item?.status, "measured", JSON.stringify(item));
|
||||
return (item as { value: number }).value;
|
||||
}
|
||||
|
||||
function reason(item: Measured | undefined): string {
|
||||
assert.equal(item?.status, "unmeasured", JSON.stringify(item));
|
||||
return (item as { reason: string }).reason;
|
||||
}
|
||||
|
||||
test("运行定义：一个容器，总时限和核数照任务，实验按顺序、每个分到总时限 ÷ 实验数，种子经环境变量交给 harness", () => {
|
||||
const definition = diagnosisRun(task, digest, "seed-x", ["a", "b", "c"]);
|
||||
assert.equal(definition.wallSeconds, TOTAL_SECONDS);
|
||||
assert.equal(definition.cores, 1);
|
||||
assert.equal(definition.env.INFORSI_SEED, "seed-x");
|
||||
assert.deepEqual(definition.command, [
|
||||
"python", "-B", "/work/harness-diagnosis.py", "--program", "initial.py", "--baseline", "baseline-artifact",
|
||||
"--answers", "answers", "--seconds-each", "8", "--report", "diagnosis.json", "--experiments", "a", "b", "c",
|
||||
]);
|
||||
});
|
||||
|
||||
test("原始产物：只读地放进实验员工作目录，存版本时被跳过；运行目录里同样有，并带全部实验登记的指标说明", (t) => {
|
||||
assert.deepEqual(diagnosisUnversionedNames(task), [...task.facts.unversionedNames, "baseline-artifact"]);
|
||||
const workDir = tempDir(t, "work");
|
||||
placeBaselineArtifact(task, baselineRunDir(t), workDir);
|
||||
const placed = join(workDir, "baseline-artifact", "solution.json");
|
||||
assert.equal(statSync(placed).mode & 0o777, 0o444);
|
||||
assert.deepEqual(JSON.parse(readFileSync(placed, "utf8")), rowLayout(1 / 64, 1 / 32, 7 / 64));
|
||||
|
||||
const runDir = tempDir(t, "run");
|
||||
prepareDiagnosisRun(task, programDir(t), baselineRunDir(t), runDir, EXPERIMENTS);
|
||||
for (const name of ["initial.py", "harness.py", "harness-diagnosis.py", "baseline-artifact/solution.json", "metrics-spec.json"]) {
|
||||
assert.ok(existsSync(join(runDir, name)), name);
|
||||
}
|
||||
assert.ok(statSync(join(runDir, "answers")).isDirectory());
|
||||
const spec = JSON.parse(readFileSync(join(runDir, "metrics-spec.json"), "utf8")) as HypothesisMetric[];
|
||||
assert.deepEqual(spec.map((s) => s.id), ["d-loose.note", "d-same.note"]);
|
||||
});
|
||||
|
||||
test("从头到尾：六个实验一个容器按顺序跑，崩了、超时、改只读文件的各记各的，不影响别人；逐个单独判分", async (t) => {
|
||||
const baseline = baselineRunDir(t);
|
||||
const runDir = tempDir(t, "e2e-run");
|
||||
prepareDiagnosisRun(task, programDir(t), baseline, runDir, EXPERIMENTS);
|
||||
const ids = EXPERIMENTS.map((e) => e.id);
|
||||
const process = await run(t, "e2e", diagnosisRun(task, digest, "inforsi-test-seed", ids), runDir);
|
||||
assert.equal(process.ending, "exited", JSON.stringify(process));
|
||||
assert.equal((process as { exitCode: number }).exitCode, 0);
|
||||
|
||||
const judgeDir = tempDir(t, "e2e-judge");
|
||||
const step = prepareDiagnosisJudging(task, runDir, judgeDir, digest, process, EXPERIMENTS);
|
||||
assert.equal(step.status, "validate");
|
||||
if (step.status !== "validate") return;
|
||||
const validator = await run(t, "e2e-validate", step.definition, judgeDir);
|
||||
const judged = judgeDiagnosis(task, runDir, judgeDir, process, validator, EXPERIMENTS);
|
||||
const by = Object.fromEntries(judged.experiments.map((e) => [e.id, e]));
|
||||
assert.deepEqual(judged.experiments.map((e) => e.id), ids);
|
||||
|
||||
assert.equal(by["d-loose"]!.ending, "finished");
|
||||
assert.equal(by["d-loose"]!.verdict, "valid");
|
||||
assert.equal(value(by["d-loose"]!.score), 0.203125);
|
||||
assert.equal(value(by["d-loose"]!.metrics["d-loose.note"]), 0.5);
|
||||
assert.equal(value(by["d-loose"]!.metrics["legalization-deduction"]), 0);
|
||||
assert.ok(value(by["d-loose"]!.wallSeconds) > 0 && value(by["d-loose"]!.wallSeconds) < 4);
|
||||
|
||||
// 读到了原始产物：原样返回，成绩就是那份产物的半径和。指标没写，记测不到，也没有串进上一个实验写的值。
|
||||
assert.equal(by["d-same"]!.verdict, "valid");
|
||||
assert.equal(value(by["d-same"]!.score), 0.40625);
|
||||
assert.match(reason(by["d-same"]!.metrics["d-same.note"]), /metrics\.json 不存在/);
|
||||
|
||||
assert.equal(by["d-crash"]!.ending, "crashed");
|
||||
assert.equal(by["d-crash"]!.verdict, "invalid");
|
||||
assert.match(reason(by["d-crash"]!.score), /非零退出码/);
|
||||
assert.match(reason(by["d-crash"]!.metrics["raw-score"]), /没有跑校验器/);
|
||||
|
||||
// 超时：被杀在分到的 4 秒上，用时不短于 4 秒；后面的实验照常跑完。
|
||||
assert.equal(by["d-slow"]!.ending, "timeout");
|
||||
assert.match(by["d-slow"]!.verdictReason, /超过分到的时间 4 秒/);
|
||||
assert.ok(value(by["d-slow"]!.wallSeconds) >= 4 && value(by["d-slow"]!.wallSeconds) < 8);
|
||||
|
||||
assert.equal(by["d-tamper"]!.ending, "crashed");
|
||||
assert.deepEqual(JSON.parse(readFileSync(join(runDir, "baseline-artifact", "solution.json"), "utf8")), rowLayout(1 / 64, 1 / 32, 7 / 64));
|
||||
|
||||
assert.equal(by["d-outside"]!.ending, "finished");
|
||||
assert.equal(by["d-outside"]!.verdict, "invalid");
|
||||
assert.match(by["d-outside"]!.verdictReason, /^校验器判不合法/);
|
||||
assert.match(reason(by["d-outside"]!.score), /^不算数/);
|
||||
});
|
||||
|
||||
/** 手写一次"只跑了一部分"的运行目录：report 里只有头几项。 */
|
||||
function partialRun(t: TestContext, report: object[]): string {
|
||||
const dir = tempDir(t, "partial");
|
||||
writeFileSync(join(dir, "diagnosis.json"), JSON.stringify(report));
|
||||
return dir;
|
||||
}
|
||||
|
||||
test("整次运行被总时限杀掉：报告里没有的实验记没轮到，写明整次运行的结局；已有记录的照常", (t) => {
|
||||
const timeout: ProcessResult = { ending: "timeout", cpuSeconds: m(23), wallSeconds: m(24), peakMemoryBytes: m(5) };
|
||||
const two = EXPERIMENTS.slice(0, 2);
|
||||
const runDir = partialRun(t, [{ id: "d-loose", status: "timeout", wall_seconds: 4.01 }]);
|
||||
const step = prepareDiagnosisJudging(task, runDir, tempDir(t, "judge"), digest, timeout, two);
|
||||
assert.equal(step.status, "judged");
|
||||
const judged = (step as { result: DiagnosisJudged }).result;
|
||||
assert.equal(judged.experiments[0]!.ending, "timeout");
|
||||
assert.equal(value(judged.experiments[0]!.wallSeconds), 4.01);
|
||||
assert.equal(judged.experiments[1]!.ending, "notRun");
|
||||
assert.match(judged.experiments[1]!.verdictReason, /没轮到：整次诊断运行超过单次时限 24 秒/);
|
||||
assert.match(reason(judged.experiments[1]!.wallSeconds), /没轮到/);
|
||||
// 没有交出答案的实验，judgeDiagnosis 不要校验器的结果。
|
||||
assert.deepEqual(judgeDiagnosis(task, runDir, tempDir(t, "judge2"), timeout, null, two), judged);
|
||||
});
|
||||
|
||||
test("运行正常退出、报告里却缺实验：报错停下，不当成候选的失败", (t) => {
|
||||
const exited: ProcessResult = { ending: "exited", exitCode: 0, cpuSeconds: m(1), wallSeconds: m(2), peakMemoryBytes: m(5) };
|
||||
const runDir = partialRun(t, []);
|
||||
mkdirSync(join(runDir, "answers"));
|
||||
assert.throws(() => prepareDiagnosisJudging(task, runDir, tempDir(t, "judge"), digest, exited, EXPERIMENTS), /没有实验 d-loose/);
|
||||
});
|
||||
@@ -102,6 +102,14 @@ const samples: { [K in SubmissionKind]: Submission<K> } = {
|
||||
],
|
||||
},
|
||||
claim: { choices: ["researcher-2-e1", "researcher-1-e2"] },
|
||||
diagnosisProposal: {
|
||||
explanations: [{ body: "卡在起点：……；若换成明显更强的起点分数不涨，这条解释就不成立" }],
|
||||
experiments: [{ name: "更强的起点", spot: "solve.py 的起点生成", body: "换成……，交出答案", metrics: metrics.defined }],
|
||||
},
|
||||
diagnosisPrediction: { predictions: [{ explanation: "r2-dx1", experiment: "d1", range }], body: "理由" },
|
||||
diagnosisImplementation: { locations: [{ experiment: "d1", locations: ["solve.py:10-40"] }], concerns: "无", body: "说明" },
|
||||
diagnosisRepair: { locations: [{ experiment: "d1", locations: ["solve.py:10-40"] }], concerns: "无", body: "按核验意见补上" },
|
||||
diagnosisVerification: { experiments: [{ experiment: "d1", followed: true, nothingElse: true, where: "solve.py:10-40" }], body: "无" },
|
||||
proposal: { factors, safeguards: ["final-shrink"], metrics, authorEstimate: range, body },
|
||||
firstPrediction: { range, body: "理由" },
|
||||
critique: { target: "p2", body: "（推测）扰动可能和起点数混杂" },
|
||||
|
||||
@@ -260,3 +260,23 @@ test("比较的行号：改一行、加几行、只删不加，新旧两边的
|
||||
// 给模型看的差异原文带 3 行上下文：三处改动相隔不到 6 行，合成一块。
|
||||
assert.match(result.diff, /^@@ -1,10 \+1,12 @@\n a\n-b\n\+B\n c\n d\n e\n\+X\n\+Y\n\+Z\n f\n g\n-h\n i\n j\n$/m);
|
||||
});
|
||||
|
||||
test("诊断程序（任务书 18）：基线代码取出后放进只读的原始产物、改成诊断程序再存，产物被跳过，比较只看程序的改动，取出再存号不变", (t) => {
|
||||
const repo = newRepo(t);
|
||||
const baseline = save(repo, makeDir(t, { "initial.py": "a\nb\n", "evaluate.py": "e\n" }), []);
|
||||
const workDir = checkout(t, repo, baseline);
|
||||
// 原始产物放进实验员工作目录，设成只读（0444），与 tasks 的 placeBaselineArtifact 一样。
|
||||
mkdirSync(join(workDir, "baseline-artifact"));
|
||||
writeFileSync(join(workDir, "baseline-artifact", "solution.json"), "{}");
|
||||
chmodSync(join(workDir, "baseline-artifact", "solution.json"), 0o444);
|
||||
writeFileSync(join(workDir, "initial.py"), "a\nb\nexperiment\n");
|
||||
const excluded = ["__pycache__", "solution.json", "baseline-artifact"];
|
||||
const program = save(repo, workDir, excluded);
|
||||
assert.notEqual(program, baseline);
|
||||
const comparison = compareVersions(repo, baseline, program);
|
||||
assert.deepEqual(comparison.files.map((f) => f.path), ["initial.py"]);
|
||||
assert.deepEqual((comparison.files[0] as Extract<FileChange, { binary: false }>).hunks, [{ from: { after: 2 }, to: { start: 3, end: 3 } }]);
|
||||
const again = checkout(t, repo, program);
|
||||
assert.equal(existsSync(join(again, "baseline-artifact")), false);
|
||||
assert.equal(save(repo, again, excluded), program);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user