diff --git a/docs/结构.md b/docs/结构.md index f43582c..8658935 100644 --- a/docs/结构.md +++ b/docs/结构.md @@ -158,7 +158,7 @@ Jev 是一个判别器，和程序检查地位相同，不单独成一种记录：它的原始概率和结论，记在它判断的那件事里（第 3 种的提交和退回、第 8 种的判重合并）。 -**写入时由 ledger 拒绝的几种情况**：阶段已关闭还要写提交；要替换的那一份不是本人在这个阶段刚交的；没有"确认启动"就写运行结果（"确认没启动"这种结果除外）；这次消融已经有格启动了才冻结解释。 +**写入时由 ledger 拒绝的几种情况**：阶段已关闭还要写提交；要替换的那一份不是本人在这个阶段刚交的；没有"确认启动"就写运行结果（"确认没启动"这种结果除外）；这次消融已经有格启动了才冻结解释；这次诊断运行已经确认启动了才冻结诊断预测。 --- @@ -245,8 +245,8 @@ Jev 是一个判别器，和程序检查地位相同，不单独成一种记录 **名单** - `ROLES`、`Role`：三个角色，研究者、实验员、核验员。 -- `STAGES`、`Stage`：有人交提交的各个阶段，按流程先后排；程序自己做的步骤不在里面。 -- `SUBMISSION_KINDS`、`SubmissionKind`：每种提交属于哪个阶段、由哪个角色交，各模块都从这张表取。 +- `STAGES`、`Stage`：有人交提交的各个阶段，按流程先后排；程序自己做的步骤不在里面。诊断有四个：诊断解释与实验、诊断预测、诊断实现（含修复）、诊断核验（任务书 18）。 +- `SUBMISSION_KINDS`、`SubmissionKind`：每种提交属于哪个阶段、由哪个角色交，各模块都从这张表取。诊断加五种：`diagnosisProposal`、`diagnosisPrediction`、`diagnosisImplementation`、`diagnosisRepair`、`diagnosisVerification`；实现和修复同一个字段定义，写法同正向的 `implementation`、`repair`。 **指标** @@ -276,7 +276,7 @@ Jev 是一个判别器，和程序检查地位相同，不单独成一种记录 **Jev** - `JevJudgment`（字段定义）：Jev 对一个问题的一次判断：问题名、问题全文的哈希（措辞、示例、阈值一起算）、阈值、原始概率、结论（概率是否超过阈值）、Jev 返回的请求编号。jev 交出它，ledger 在提交和退回记录里原样存它，两个模块互不引用，所以放在这里。 -- `Checklist`、`ChecklistOf`（类型）：对号清单，materials 为一次任务算出、submissions 拿来判收不收；按提交种类区分，每种只放它要用的（入口和讨论的入口个数范围；认领的入口编号和至少排几个；提案的 M、因子数上限、能延续的因子、能引用的假设专用指标（全名）、已占用的指标编号（任务的指标和两个保留键）；两次预测的候选；挑刺的原提案和它的因子；回应、统一的挑刺编号（统一另有被合并的原提案）；实现、修复、核验的条款、因子、保障（核验另有没有归属的改动）；解释的格和 M；消融实现的格、要关的因子、全部因子；消融核验的格和全部因子；收尾无）。用户 2026-10-03 决定：vocab 只放这一个，任务材料的其余类型放 materials。 +- `Checklist`、`ChecklistOf`（类型）：对号清单，materials 为一次任务算出、submissions 拿来判收不收；按提交种类区分，每种只放它要用的（入口和讨论的入口个数范围；认领的入口编号和至少排几个；提案的 M、因子数上限、能延续的因子、能引用的假设专用指标（全名）、已占用的指标编号（任务的指标和两个保留键）；两次预测的候选；挑刺的原提案和它的因子；回应、统一的挑刺编号（统一另有被合并的原提案）；实现、修复、核验的条款、因子、保障（核验另有没有归属的改动）；解释的格和 M；消融实现的格、要关的因子、全部因子；消融核验的格和全部因子；收尾无；诊断解释与实验的解释上限 M、实验上限 A 和已占用的指标编号；诊断预测的自己的解释编号和合并后的实验编号；诊断实现、修复、核验的选中实验编号）。用户 2026-10-03 决定：vocab 只放这一个，任务材料的其余类型放 materials。 ### rules @@ -376,6 +376,21 @@ Jev 是一个判别器，和程序检查地位相同，不单独成一种记录 - `BetResult`（类型）：押的因子的结果：更新后的成分权重、权重是否无从更新、各因子的方向收益（实测换算的或估计的）、挑出的因子及其预测收益、押或不押的两种情况、关掉反而更好的格。 - `chooseBet(input)`：押的因子，用有效格的实测更新各成分的权重，再挑方向收益最大的因子；有格关掉反而更好或估计收益不为正时不押，实测落在所有成分之外时保持起始权重（6.8）。 +**诊断（6.12，4.3 诊 3、诊 4）** + +成分、权重更新、精确积分、部分格网格、价值减成本的公式只在 `value.ts` 写一份，消融和诊断共用，差别只在"实测 → 收益"的换算：消融是撑几成乘剩余空间 h，没有"都不走"；诊断是改进量 x − b，h 取 1，另有收益为 0 的"都不走"（`floor = 0`）。消融的数字逐位不变（任务书 18 第 1.2 节）。 + +- `DiagnosisExplanation`（类型）：一条诊断解释，带作者和它对每个合并后的实验给的三个数。 +- `DiagnosisModelInput`、`DiagnosisModel`（类型）：成分模型的输入和结果（因子换成实验、撑几成换成改进量）。 +- `diagnosisModel(input)`：建成分模型，每条解释一个成分、开放预留一个成分，每个实验自己的分布和开放预留（6.3"诊断"、6.12"成分与权重"）。 +- `DiagnosisParams`、`DiagnosisCombo`（类型）：余额、每批上限、u、衰减和 D；一个实验组合的 VOI、R̂、m′、价值、成本、净值。 +- `evaluateDiagnosisCombos(model, params)`：列出可选组合并算价值、成本、净值：余额不到 2 次没有组合；实验数不超过 D 时只有"全做"，超过 D 时是所有恰好 D 个实验的组合。R̂ = ⌈(余额 − 1) ÷ 每批上限⌉，成本按一次运行算，ĝ₀ = max(0, max_d E(g_d))。 +- `DiagnosisRuleInput`、`DiagnosisDecision`、`SkippedReason`（类型）：选诊断实验的输入和结果，结果含 6.12 要求附上的全部数字：参数、成分与起始权重、各实验的事前期望收益、ĝ₀、全部组合的价值成本净值、做不做、选中的实验及其净值、每个实验做不做和没做的原因（放不下、信息价值不足以抵过成本）以及含它的净值最大组合。 +- `selectDiagnosis(input)`：选诊断实验规则"信息价值减成本"：取净值最大且 > 1e-12 的组合，差不到 1e-12 的按实验编号比（和选格同一条挑法），都不大于 1e-12 时不诊断。 +- `ExperimentOutcome`（类型）：一个实验的结局：有效分数、证据不足、没做（和消融一格同形）。 +- `SettleDiagnosisInput`、`DiagnosisDirection`（类型）：一次诊断结算的输入；诊断推出的方向（更新后的权重、各实验的收益、收益最大的实验、有没有指出方向）。 +- `settleDiagnosis(input)`：一次诊断的结算：每个实验的收益和合并范围，有效实验的世界比特、惊讶、分歧量；每条解释在每个有效实验上的确认或否定（用 `judgeExplanations`）；诊断推出的方向，最大收益不为正时 `found` 为 false。证据不足和没做的实验不判、不记账。 + **信息账（6.9）** 这一组是方法里"唯一的尺子"，不随选批、选格规则变，一律在分数原值上从本批基线 b 起按格宽切小格。 @@ -397,9 +412,9 @@ Jev 是一个判别器，和程序检查地位相同，不单独成一种记录 - `runBalance(total, runs)`：从运行记录数出已扣次数、占用名额和余额；确认启动的扣一次，已预留的先占着名额，确认没启动的退还。只数运行次数，不算计算时间。 - `forwardBatchSize(batchCap, balance)`：本轮正向批量 k = min(每批上限, 余额)。 - `ablationCellLimit(balance)`：消融最多测"余额 − 1"格，余额为 0 或 1 时是 0。 -- `Phase`（类型）：正向或消融。 +- `Phase`（类型）：正向、诊断或消融。 - `Tally`（类型）：一个人的战绩：总数、中位数偏高几次、偏低几次、被穿过上沿几次、下沿几次、包住几次、落点四段计数。 -- `trackRecord(samples)`：按研究者、按正向与消融分开数战绩，只用来显示，不改任何权重（6.9、6.10）。 +- `trackRecord(samples)`：按研究者、按正向、诊断与消融分开数战绩，结果按研究者编号、正向、诊断、消融排，只用来显示，不改任何权重（6.9、6.10）。 - `authorVsMerged(items, direction)`："作者自评 vs 大家合并"，给出作者比大家乐观多少，以及实测落在两个范围的哪一段；只收已经跑过、有有效成绩的候选（6.10 第 4 条）。 **入口分配、种子与规则登记表（4.2、第九部分、第十一部分）** @@ -411,8 +426,10 @@ Jev 是一个判别器，和程序检查地位相同，不单独成一种记录 - `seededSample(items, size, seed)`：用种子从一组编号里不放回地均匀抽 size 个，结果只取决于种子和编号集合，与传入次序无关；候选和因子都这样抽。 - `BATCH_RULES`、`BatchRuleName`：选批规则登记表，名字 → 函数和版本号，两条：批内最好期望、均匀随机。 - `CELL_RULES`、`CellRuleName`：选格规则登记表，四条：信息价值减成本、全测、随机抽同样多的格、都不测。 +- `DIAGNOSIS_RULES`、`DiagnosisRuleName`：选诊断实验规则登记表，只有一条：信息价值减成本（6.12），配置里不选，步骤清单的 `selectDiagnosis` 一步带它的名字和版本。 - `runBatchRule(name, input)`：按名字跑一条选批规则，结果带上规则名和版本，flow 写决定记录时从这里取。 - `runCellRule(name, input)`：按名字跑一条选格规则，结果带上规则名和版本。 +- `runDiagnosisRule(name, input)`：按名字跑选诊断实验规则，结果带上规则名和版本。 ### ledger @@ -420,9 +437,9 @@ Jev 是一个判别器，和程序检查地位相同，不单独成一种记录 **打开与写入** -- `Ledger.open(path)`：打开一份研究的账本，新文件就建表，旧文件核对表结构版本号（现为 4），对不上就报错。 +- `Ledger.open(path)`：打开一份研究的账本，新文件就建表，旧文件核对表结构版本号（现为 5），对不上就报错。 - `ledger.close()`：关闭数据库。 -- `ledger.append(kind, content)`：追加一条记录并返回它。先按种类检查内容，再做第四部分说的四种拒收检查（第 1 条是只收开着的阶段的提交，即有开阶段记录、没有关阶段记录），都在同一个事务里。有自己编号、按定义只发生一次的记录按编号加了唯一约束，重复写就报错：阶段开、阶段关、运行登记、确认启动（"结果是没启动"也算在这一条里，一次运行两样只能有一样）、运行结果、模型调用开始、模型调用结束、消融结算（按轮次，一轮一条）；其余记录照写。记录之间的先后（例如结果前要先有登记）不查。写提案时，作者自评从提案里拆出来，在同一个事务里另存成一条封存。 +- `ledger.append(kind, content)`：追加一条记录并返回它。先按种类检查内容，再做第四部分说的五种拒收检查（第 1 条是只收开着的阶段的提交，即有开阶段记录、没有关阶段记录；第 5 条是这一轮的诊断运行已经确认启动，就不能再关这一轮的诊断预测阶段），都在同一个事务里。有自己编号、按定义只发生一次的记录按编号加了唯一约束，重复写就报错：阶段开、阶段关、运行登记、确认启动（"结果是没启动"也算在这一条里，一次运行两样只能有一样）、运行结果、模型调用开始、模型调用结束、消融结算、诊断结算（都按轮次，一轮一条）；其余记录照写。记录之间的先后（例如结果前要先有登记）不查。写提案时，作者自评从提案里拆出来，在同一个事务里另存成一条封存。 - `ledger.transaction(fn)`：里面的几次追加要么全部写进去，要么一条都不写；fn 必须是同步函数，返回值类型排除了 Promise，异步函数在类型检查时就被挡住；不能嵌套。 **读取** @@ -435,7 +452,7 @@ Jev 是一个判别器，和程序检查地位相同，不单独成一种记录 **类型** -- `Kind`：记录的种类，共 19 种：`config`（研究配置）、`stageOpen` / `stageClose`（阶段开、关）、`submission`（提交的每一版，外层带收下时用到的 `JevJudgment` 数组，没用到 Jev 的为 null）、`rejection`（退回，带对象 `target`（与提交的 target 同义，没有对象的写 null；同一人在同一阶段可能对几个对象各做一次同种任务，materials 按它找本人这个对象的上一次退回）、原样内容和理由；Jev 退回的，原样存这次用到的一条或几条 `JevJudgment`，程序检查退回的这一项写 null）、`note`（私人笔记，每次修改追加一条，以最后写入的一条为准，不带版本号）、`protocol`（冻结协议）、`seal` / `unseal`（作者自评的封存、解封，都用记录序号指明是哪一条）、`code`（冻结代码）、`runPlanned` / `runStarted` / `runResult`（运行的登记要跑、确认启动、结果）、`decision`（决定）、`settlement`（一轮的消融结算，按轮次唯一）、`callStarted` / `callUsage` / `callEnded`（模型调用的开始，附这次任务的全部材料；每次回复的用量；结束，带墙钟时间（秒，`Measured`：中断后恢复时补写的结束记录，结局记出错、原因写"进程中断"，墙钟时间记测不到），结局与 agents 的会话结局同名：交了、没交（带原因）、出错（带原因）、被停下）、`event`（事件）。 +- `Kind`：记录的种类，共 20 种：`config`（研究配置）、`stageOpen` / `stageClose`（阶段开、关）、`submission`（提交的每一版，外层带收下时用到的 `JevJudgment` 数组，没用到 Jev 的为 null）、`rejection`（退回，带对象 `target`（与提交的 target 同义，没有对象的写 null；同一人在同一阶段可能对几个对象各做一次同种任务，materials 按它找本人这个对象的上一次退回）、原样内容和理由；Jev 退回的，原样存这次用到的一条或几条 `JevJudgment`，程序检查退回的这一项写 null）、`note`（私人笔记，每次修改追加一条，以最后写入的一条为准，不带版本号）、`protocol`（冻结协议）、`seal` / `unseal`（作者自评的封存、解封，都用记录序号指明是哪一条）、`code`（冻结代码）、`runPlanned` / `runStarted` / `runResult`（运行的登记要跑、确认启动、结果；登记的用途有基线、正向、消融、诊断四种）、`decision`（决定）、`settlement`（一轮的消融结算，按轮次唯一）、`diagnosisSettlement`（一轮的诊断结算，按轮次唯一，和消融结算各管各的）、`callStarted` / `callUsage` / `callEnded`（模型调用的开始，附这次任务的全部材料；每次回复的用量；结束，带墙钟时间（秒，`Measured`：中断后恢复时补写的结束记录，结局记出错、原因写"进程中断"，墙钟时间记测不到），结局与 agents 的会话结局同名：交了、没交（带原因）、出错（带原因）、被停下）、`event`（事件）。 - `AppendKind`：调用方能追加的种类，除了 `seal`（封存只由 ledger 在写提案时一并写）。 - `Content`：某种记录存下的内容。 - `Input`：追加时交的内容；只有提交和存下的不同：提案带着作者自评。 @@ -454,7 +471,7 @@ Jev 是一个判别器，和程序检查地位相同，不单独成一种记录 **字段定义** -- `ResearchConfig`（字段定义）：research.json 的全部字段，全部必填，不认识的字段报错。每个字段带中文说明（`description`）和类别（`category`：用户配置或方法参数），配置对话照着它问，不另写一份。`models` 里除了各角色的模型，还有 Jev 用的模型名（`models.jev`，用户配置）；Jev 的密钥不进配置，从环境变量读。流程是两选一：完整流程写四个开关、选批规则、它才用得到的 λ、u、衰减；朴素循环只写 `preset`，别的都写不进去。 +- `ResearchConfig`（字段定义）：research.json 的全部字段，全部必填，不认识的字段报错。每个字段带中文说明（`description`）和类别（`category`：用户配置或方法参数），配置对话照着它问，不另写一份。`models` 里除了各角色的模型，还有 Jev 用的模型名（`models.jev`，用户配置）；Jev 的密钥不进配置，从环境变量读。流程是两选一：完整流程写开关（含诊断：关，或开并写每人每轮诊断实验上限 A 和一次诊断最多做几个实验 D，两个都是方法参数）、选批规则、它才用得到的 λ、u、衰减；朴素循环只写 `preset`，别的都写不进去。 - 研究编号 `studyId` 只许小写字母、数字和 `._-`，首字符是小写字母或数字：它会进 Docker 镜像名，镜像名只认小写。（来源：用户 2026-10-03 同意，任务书 13） - `task`、`promptDir` 必须是绝对路径：从别的目录接着跑一份研究时，相对路径会读到别处，而任务目录没有哈希核对挡着。（来源：用户 2026-10-03 同意，任务书 13） - `FIELD_CATEGORIES`、`FieldCategory`：字段的两种类别，用户配置、方法参数。 @@ -467,7 +484,7 @@ Jev 是一个判别器，和程序检查地位相同，不单独成一种记录 - `Step`（类型）：一步：编号、规则（只有选批、选格有，其余为空）、执行或跳过及原因。 - `RuleRef`（类型）：决策点用的规则，名字和版本。 - `STUDY_STEPS`、`StudyStepId`：整份研究的步骤，按先后排。 -- `ROUND_STEPS`、`RoundStepId`：每一轮的步骤，按先后排；比第三部分拆得更细（例如"回应与统一"和"协议冻结"分成两步，挑刺与回应关掉时协议照样冻结）。 +- `ROUND_STEPS`、`RoundStepId`：每一轮的步骤，按先后排；诊断七步（`diagnosisProposal`、`diagnosisDedupe`、`diagnosisPrediction`、`selectDiagnosis`、`diagnosisImplementation`、`diagnosisRun`、`diagnosisSettle`）排在 `carryOverPool` 与 `proposal` 之间，`selectDiagnosis` 带规则名和版本；诊断关掉时七步都按配置跳过，消融关掉时 `explanation` 也跳过；比第三部分拆得更细（例如"回应与统一"和"协议冻结"分成两步，挑刺与回应关掉时协议照样冻结）。 ### runner @@ -528,13 +545,14 @@ Jev 是一个判别器，和程序检查地位相同，不单独成一种记录 下面是 `src/materials/index.ts` 导出的全部内容。谁能看什么每种任务一个函数，对着方法.md 第三部分、4.2、4.3 写，不写通用判定（用户 2026-10-02）。materials 只读账本，不写；任务材料由 flow 整份存进"模型调用开始"记录。依赖只有 ledger、rules、vocab；代码比较结果、运行产物、提示词由 flow 传进来。 -- `buildMaterial(req)`：生成一份任务材料：给模型看的各节、可查询的记录、拼好的第一条消息、对号清单、能读的代码版本。研究者的材料按"任务说明 → 你自己的 → 公共材料 → 现在交什么"排；实验员、核验员的按"任务说明 → 分给你的材料 → 现在交什么"排，不给信念本和成绩。同一人重新发起同一任务时，最前面放上一次被退回的原因：按阶段、作者、提交种类、对象找本人最后一条退回记录。"现在交什么"由 vocab `SUBMISSION_SCHEMAS` 的字段说明生成。几种任务的取法： +- `buildMaterial(req)`：生成一份任务材料：给模型看的各节、可查询的记录、拼好的第一条消息、对号清单、能读的代码版本。研究者的材料按"任务说明 → 你自己的 → 公共材料 → 现在交什么"排；实验员、核验员的按"任务说明 → 分给你的材料 → 现在交什么"排，不给信念本和成绩。同一人重新发起同一任务时，最前面放上一次被退回的原因：按阶段、作者、提交种类、对象找本人最后一条退回记录。"现在交什么"由 vocab `SUBMISSION_SCHEMAS` 的字段说明生成。`StudyFacts` 多一个 `diagnosisExperimentLimit`（每人每轮诊断实验上限 A，诊断关掉为 null），诊 1 的对号清单用它。几种任务的取法： - 讨论、认领：展示讨论原文时，原文引用的入口编号已经作废的（讨论时整份替换过清单），在它后面就地标"（已作废）"，读的人才分得清哪些编号还有效（用户 2026-10-03 同意，任务书 14 第 2.5 节）。前后紧挨字母、数字或"-"的不算，`e-x-1` 不会匹配到 `e-x-12` 里。 - 第二次预测：冻结协议不带答复，"挑刺与答复"从这个候选已公开的最后一份回应或统一里读（回应另给答复正文）；留池候选读它冻结那一轮的（用户 2026-10-03 定）。 - 核验：不放 Jev 对这份实现的判断。能走到核验说明 Jev 已判过，Jev 是判别条件，不是给核验员的提示。 - 消融实现：这一格已有公开的消融核验（核验不通过、重做）时，和修复一样从上一次交的这一格代码接着改（用户 2026-10-03 定）：工作目录写"上一次交的这一格代码的副本"，能读的版本里加上它，完整代码照旧列出作对照；另放"退回意见"：最近一次核验里关错了的因子和核验员写的依据，写法同修复的退回意见。 + - 诊断（方法.md 第三部分、4.3 诊 1–诊 4）：诊 1 放当前基线的代码、产出它的那次运行的结果和原始答案（产物放进查询，flow 经 `artifacts` 传进来）、信念本、上一轮收尾，第 1 轮的第零轮记录在研究记录里；诊 2 放合并后的诊断实验（标明作者）和全部诊断解释，自己的解释另列，别人在这一步交的数一概不读；诊断实现、修复、核验只看选中的实验原文和当前基线代码（实现、修复另告诉原始产物已只读放进工作目录，位置在任务说明里；修复另放退回意见，只修没通过的实验；核验放当前基线到诊断程序的改动和位置表），不给信念本和成绩。从 ② 提案起的全部研究者任务，公共材料里在"基线"之后加"本轮诊断结果"一节（每个实验的做法、实测、用时、结局，每条解释确认或否定，推出的方向；本轮没诊断写原因：余额不到 2 次、配置关掉、信息价值不抵成本、选中的实验一个都没剩下）；诊断运行在诊断结算写入后公开，诊断程序进"能读的代码"，各人给的三个数不进任何记录或查询。实现、修复正向候选的实验员，本轮有诊断时多一个只读的诊断程序目录，注明"冻结协议引用诊断实验时作参考"。信念本加"诊断历史"，战绩加"诊断"一档；诊断实验的实测不算"手里最好"（诊断运行的运行记录不带判分，逐个实验的结果只在诊断结算里）。 - 指标：冻结之后一律用全名"候选编号.短名"：已登记指标清单、对号清单的 `reusableMetrics`、实验员材料的"要实现的指标定义"（候选代码在结果文件里也用全名当键）。提案、回应、统一的对号清单里，已占用的指标编号只有任务的指标、`answer-path` 和 `safeguard-triggers` 两个保留键（用户 2026-10-03 定）；已登记的全名带 `.`，新登记的短名不带，不用列进来。 -- `Board`（类）：materials 读账本的那一份，构造参数是账本记录数组。flow 的"下一步做什么"是只拿得到记录数组的纯函数，它和 flow 的三个程序步骤都经这一份读账本，不另写一份（任务书 14 第 2.4 节）。flow 用到的：`stages`、`stage`（阶段的轮次、名单、关没关）、`published`（已公开的某种提交，最后一版为准）、`decision`（某轮某种决定）、`proposal`、`proposalsOf`、`authorsOf`（候选合并了哪些原提案、作者按编号排）、`answerOf`（冻结出候选的最后一份回应或统一）、`protocol`、`protocols`、`carried`（留池候选）、`baselineOf`（第 r 轮的本批基线）、`baselineScore`、`runStates`。任务书 14 写的是导出 `baselineOf`、`baselineScore` 两个函数，实际导出了整个类，原因是纯函数只拿得到记录数组，要用的远不止这两样（这一处偏离待用户确认）。 +- `Board`（类）：materials 读账本的那一份，构造参数是账本记录数组。flow 的"下一步做什么"是只拿得到记录数组的纯函数，它和 flow 的三个程序步骤都经这一份读账本，不另写一份（任务书 14 第 2.4 节）。flow 用到的：`stages`、`stage`（阶段的轮次、名单、关没关）、`published`（已公开的某种提交，最后一版为准）、`decision`（某轮某种决定）、`proposal`、`proposalsOf`、`authorsOf`（候选合并了哪些原提案、作者按编号排）、`answerOf`（冻结出候选的最后一份回应或统一）、`protocol`、`protocols`、`carried`（留池候选）、`baselineOf`（第 r 轮的本批基线）、`baselineScore`、`runStates`；诊断另有：`runStatesBefore(round)`（这一轮之前各次运行的状态，算诊断开始时的余额）、`diagnosisSettlement(round)`、`diagnosisSettlements()`、`diagnosisDone(round)`（提案阶段开了，就是诊断这一段走完了或本轮不诊断）、`diagnosisExplanations(round)`（已公开的诊断解释：编号、作者、正文）、`diagnosisExperiments(round)`（合并后的诊断实验：`MergedExperiment`，编号、全部作者、正文取作者里编号最小的那一份、登记的指标已改全名"实验编号.短名"）。任务书 14 写的是导出 `baselineOf`、`baselineScore` 两个函数，实际导出了整个类，原因是纯函数只拿得到记录数组，要用的远不止这两样（这一处偏离待用户确认）。 - `protocolMetrics(board, protocol)`：把冻结协议自定义及引用的指标合成原登记定义，实验员材料与 flow 问 Jev、正式运行共用（来源：任务书 16 第 2.2、2.4 节）。 - `finalEntries(ledger)`：按账本取的最终有效的入口（各人最后一版入口清单里的每个入口，带作者和正文）；flow 分配入口时用，和认领的对号清单是同一份（任务书 12 第三部分）。 - `queryMaterial(material, id)`：只在这份材料的记录里按编号找，交回全文；找不到交回 null（agents 的查询回调据此答"看不到"）。 @@ -542,12 +560,13 @@ Jev 是一个判别器，和程序检查地位相同，不单独成一种记录 - `beliefBook(ledger, study, round)`：信念本全文（方法.md 6.10 五条），数字全部用 rules 现算，只写已公开的记录；`round` 是读者所在轮次，导出时写 null。选批历史的每次运行后面原样写出任务的附加记录（判分结果的 `extra`，合法化扣掉的分在里面，字段由任务定）。消融格记"证据不足"时，有运行的原因取判分，没有运行的（核验不通过、重做次数用完）原因取这一格的 `CELL_NOT_RUN` 事件。 - `TaskMaterial`、`MaterialSection`、`MaterialRecord`、`ReadableVersion`（类型）：任务材料，以及其中的一节、一条可查询记录（编号、标题、全文）、一个能读的代码版本（版本号、说明、取出目录）。 - `TaskRequest`、`StudyFacts`、`CodeChanges`、`Prompts`、`PromptFile`（类型）：flow 传进来的东西，全部必填；`CodeChanges` 与 workspace 的比较结果同形（照 jev `codeHunks` 的先例，不引用 workspace）。 -- 账本里只存成 Json 的内容的形状（用户 2026-10-03 决定 2：materials 定义，flow 照着写，ledger 不改）：`StudyRecord`（研究配置记录：配置、步骤清单、提示词每个文件的哈希、镜像编号、任务事实、种子；配置和任务事实存 Json，flow 读回时用 `checkConfig`、`checkTaskFacts` 再检查，任务书 12 第三部分）、`FrozenProtocol`（冻结协议：方法.md 4.3 ⑥ 列的因子、保障、指标、正文、实现条款，加轮次、出发的基线、合并了哪些原提案；不带答复，主体由 submissions 的 `protocolBody` 拼，指标编号是全名）、`LocationTable`（代码记录的位置表）、`DecisionKind`、`DECISION_KINDS`、`DecisionShapes`（六种决定：入口分配、判重合并、选批、采纳、选格、押的因子，输入输出用 rules 的类型；没采纳时也写一条采纳决定）、`DedupOutput`（判重合并的输出）、`Settlement`（结算）。 +- 账本里只存成 Json 的内容的形状（用户 2026-10-03 决定 2：materials 定义，flow 照着写，ledger 不改）：`StudyRecord`（研究配置记录：配置、步骤清单、提示词每个文件的哈希、镜像编号、任务事实、种子；配置和任务事实存 Json，flow 读回时用 `checkConfig`、`checkTaskFacts` 再检查，任务书 12 第三部分）、`FrozenProtocol`（冻结协议：方法.md 4.3 ⑥ 列的因子、保障、指标、正文、实现条款，加轮次、出发的基线、合并了哪些原提案；不带答复，主体由 submissions 的 `protocolBody` 拼，指标编号是全名）、`LocationTable`（代码记录的位置表）、`DecisionKind`、`DECISION_KINDS`、`DecisionShapes`（八种决定：入口分配、判重合并、选批、采纳、选格、押的因子、诊断实验判重合并（`diagnosisDedup`）、选诊断实验（`selectDiagnosis`），输入输出用 rules 的类型；没采纳时也写一条采纳决定）、`DedupOutput`（判重合并的输出）、`DiagnosisDedupOutput`（诊断实验判重合并的输出：合并后的每个实验由哪几个作者的哪几个实验合成，第一个的正文是合并后的正文）、`Settlement`（消融结算）、`DiagnosisSettlement`（诊断结算：运行编号、每个跑过的实验单独判分的结果（与 tasks 的 `DiagnosisJudged` 同形）、核验不通过而拿掉的实验和原因、`settleDiagnosis` 的输入输出）、`DiagnosisLocationTable`（诊断程序的位置表，每个选中的实验一项）。诊断运行的运行记录里 `judged` 写 null。 - `freezeProtocol(body, candidate, round, baseline, proposals)`：flow 写冻结协议时调用。输入 submissions 交回的冻结协议主体、候选编号、轮次、基线、合并了哪些原提案，交回 `FrozenProtocol`；新登记的假设专用指标改用全名"候选编号.短名"（例如 `r2-c1.restart-gain`），引用的本来就是全名，照抄。加前缀只在这一处：同一轮的研究者互相看不到提案，会起同一个短名、定义却不同，候选编号判重合并之后才有，所以只能在冻结时加（用户 2026-10-03 定）。 - `experimenterId(candidate)`、`verifierId(candidate)`：每个候选固定一位实验员、一位核验员，修复及再次核验沿用编号（来源：任务书 16 第 2.4 节）。 - `FORWARD_NOT_RUN`、`forwardNotRunEvent(候选, 原因)`：正向候选修复用完仍未通过核验时的“不跑”事件，`objects` 只有该候选；不扣次（来源：任务书 16 第 2.2 节）。 - 事件：`CELL_NOT_RUN`、`cellNotRunEvent(格, 原因)`：一格消融核验不通过、重做次数用完而不跑时，flow 用它写事件记录（`objects` 只有这一格），结算记证据不足（方法.md 4.3 ⑯、6.6）。 -- 编号规则：`ORIGINAL_BASELINE`、`researcherId`（第 i 位研究者，从 0 数，编号 `researcher-`，任务书 12 第三部分）、`entryId`、`proposalId`、`critiqueId`、`candidateId`、`factorId`、`cellId`、`splitCell`、`explanationId`、`callId`，以及协议内的局部编号 `factorKey`（p1…）、`clauseKey`（c1…）。入口编号按作者一直往后编，讨论时整份替换也不复用旧编号。 +- 事件：`DIAGNOSIS_NOT_RUN`、`diagnosisNotRunEvent(轮次, 原因)`：选中的诊断实验核验都没通过、修复用完而一个都不剩时，flow 写这条事件（`objects` 只有本轮诊断的编号），不跑、不扣次；材料里"本轮没有诊断"的原因从它读。 +- 编号规则：`ORIGINAL_BASELINE`、`researcherId`（第 i 位研究者，从 0 数，编号 `researcher-`，任务书 12 第三部分）、`entryId`、`proposalId`、`critiqueId`、`candidateId`、`factorId`、`cellId`、`splitCell`、`explanationId`，诊断的 `diagnosisId(轮次)`（本轮诊断：诊断程序的冻结代码、诊断运行、诊断实现与核验的对象）、`diagnosisKey(i)`（合并后第 i 个诊断实验，d1…）、`diagnosisExplanationId(轮次, 作者, i)`（诊断解释，`r1-researcher-1-dx1`），`callId`，以及协议内的局部编号 `factorKey`（p1…）、`clauseKey`（c1…）。入口编号按作者一直往后编，讨论时整份替换也不复用旧编号。 ### agents @@ -563,12 +582,15 @@ Jev 是一个判别器，和程序检查地位相同，不单独成一种记录 ### jev -下面是 `src/jev/index.ts` 导出的全部内容。九问的问法和阈值照参考项目，只改了说法和字段名；阈值没有在新项目的材料上校准过，等真实研究的数据（任务书 08 第三部分）。模型名由调用方从 config 的 `models.jev` 传入，密钥从环境变量 `TYPESAFE_API_KEY` 读进请求头。调不通时在发请求那一处有限次重试，仍不通就报错；回答读不懂报错并带请求编号。 +下面是 `src/jev/index.ts` 导出的全部内容。九问的问法和阈值照参考项目，只改了说法和字段名；诊断实验另有四问（判重一问，冻结检查三问），照判重、冻结条款的写法，阈值暂用它们的值；阈值都没有在新项目的材料上校准过，等真实研究的数据（任务书 08 第三部分、任务书 18 第三块）。模型名由调用方从 config 的 `models.jev` 传入，密钥从环境变量 `TYPESAFE_API_KEY` 读进请求头。调不通时在发请求那一处有限次重试，仍不通就报错；回答读不懂报错并带请求编号。 - `judgeSameChange(model, a, b)`：③ 判重，两条提案的因子清单是不是同一个改法，一对一次请求，交回一条 `JevJudgment`。 - `checkProtocol(model, protocol)`：⑥ 查冻结条款，三问（改动了评测、自相矛盾、太含糊）各一次请求，交回 `JevCheck`。 - `checkHunk(model, hunk, protocol)`：⑨ 查一段代码改动，五问（不固定种子的随机数、没登记就读时钟、算登记指标的方式与定义不符、比登记的停止条件更早结束、测不到的量写成 0）同一份材料一次请求，交回 `JevCheck`。对照的指标取自冻结协议，编号是全名。 +- `judgeSameExperiment(model, a, b)`：诊 2 诊断实验判重，两个实验（名称、动哪一处、怎么动）是不是同一个实验，一对一次请求，交回一条 `JevJudgment`；阈值暂用判重的 0.3，待校准。 +- `checkExperiment(model, experiment)`：诊 1 查一个诊断实验，改动了评测、自相矛盾、太含糊三问各一次请求，用冻结三问的诊断变体（材料是"一个诊断实验"，不是冻结协议），交回 `JevCheck`；阈值暂用冻结三问的值，待校准。诊断程序的代码改动直接用 `checkHunk` 的五问。 - `codeHunks(comparison)`：把 workspace 的比较结果（文件清单和差异原文）切成差异块；没有 `@@` 行的改动（二进制、只改可执行位、空文件增删）不出块，由程序检查。 +- `DedupExperiment`、`DiagnosisExperiment`（类型）：诊断实验判重和冻结检查收的输入。 - `DedupFactor`、`FrozenProtocol`、`ProtocolMetric`、`ProtocolRules`、`CodeHunk`（类型）：三个函数收的带类型输入，取自 vocab 的字段定义；材料由 jev 自己拼。`DedupFactor` 是因子的名称、做法和关键设定：问题写着"设定不同就不算同一个改法"，所以关键设定要给 Jev 看，判重的材料说明里也用一句话讲了 `keySettings` 是什么。 - `JevCheck`（类型）：每一问的判断，以及有没有任何一问判"是"。 @@ -576,12 +598,13 @@ Jev 是一个判别器，和程序检查地位相同，不单独成一种记录 下面是 `src/submissions/index.ts` 导出的全部内容。submissions 判一份提交收下还是退回：先做程序检查（vocab 字段定义管不到的跨字段规则，问题一次列全），有一项不过就退回、不问 Jev；全过了再问 Jev，几问问完、有一问判"是"就退回。它不读也不写账本，只依赖 jev 和 vocab；要 workspace、tasks、materials 算的东西由 flow 算好传进来。Jev 调不通、回答读不懂时 jev 报错，这里不接住，flow 按内部错误停下。 -- `receive(incoming)`：判一份提交，交回 `Verdict`。回应、统一先拼好冻结协议主体，程序检查、问 Jev、交回都用这同一份；回应、统一问冻结条款三问，实现、修复对每个差异块问代码改动五问，其余种类不问 Jev。Jev 退回的原因只列判"是"的几问，用 jev 的英文问题名（用户 2026-10-03 定），差异块附原文。 +- `receive(incoming)`：判一份提交，交回 `Verdict`。回应、统一先拼好冻结协议主体，程序检查、问 Jev、交回都用这同一份；回应、统一问冻结条款三问，实现、修复（含诊断程序）对每个差异块问代码改动五问，诊断解释与实验对每个实验问诊断变体的冻结三问，其余种类不问 Jev。诊断另有四条程序检查：解释 1 至 M 条、实验 1 至 A 个；诊断预测覆盖自己每条解释 × 每个合并后的实验、不多不少、三个数严格递增；诊断实验登记的指标只能是"缺了只记缺失"；诊断核验的逐个实验结果覆盖全部选中的实验。Jev 退回的原因只列判"是"的几问，用 jev 的英文问题名（用户 2026-10-03 定），差异块附原文。 - `protocolBody(from)`：拼冻结协议主体，只在这一处拼。回应：因子、保障、指标、正文四项，`changes` 写了哪项就整项换新，没写的用原提案的，实现条款取回应自己的；统一：直接用统一提交自己的字段。答复不进协议（用户 2026-10-03 定）。 - `verificationOutcome(v)`：一份已收下的核验通过没有：任何条款、因子、保障没落实，或任何没有归属的改动判为越界，就不通过；四类分开列（方法.md 4.3 ⑨）。 - `ablationOutcome(v, off, alsoOff)`：一份已收下的消融核验通过没有：要关的因子和实验员报的一起关的因子都写了"关着"、其余都写了"照留"才通过；分开列该关却照留的和该留却关了的（方法.md 4.3 ⑯）。 +- `diagnosisOutcome(v)`：一份已收下的诊断核验逐个实验通过没有：`followed` 或 `nothingElse` 任一为 false 的实验不通过，分开列没照写法做的和动了写法以外地方的（方法.md 4.3 诊 4）。 - `Incoming`（类型）：交给 `receive` 的一份提交，按提交种类区分：内容、这次任务的对号清单、提交之后才知道的东西（`KnownOf`）、Jev 的模型名。 -- `KnownOf`（类型）：提交之后才知道的东西，没有的种类为 null：提案是本人本阶段已收下几条，回应是作者那条原提案，实现和修复是 `ForwardCode`，消融实现是 `AblationCode`（交了"跑不起来"时为 null）。 +- `KnownOf`（类型）：提交之后才知道的东西，没有的种类为 null：提案是本人本阶段已收下几条，回应是作者那条原提案，实现和修复是 `ForwardCode`，消融实现是 `AblationCode`（交了"跑不起来"时为 null），诊断实现和修复也是 `ForwardCode`。 - `ForwardCode`（类型）：实现、修复的代码：存不成版本时带路径和原因；存成了带改了的不许改的文件、和本批基线比出的差异块、给 Jev 对照的协议条款。 - `AblationCode`（类型）：消融实现的代码：存不成版本时带路径和原因；存成了只带改了的不许改的文件。 - `OriginalProposal`（类型）：回应拼协议时叠在下面的原提案：因子、保障、指标、正文，作者自评不进协议。 @@ -590,7 +613,7 @@ Jev 是一个判别器，和程序检查地位相同，不单独成一种记录 ### flow -下面是 `src/flow/index.ts` 导出的全部内容，以后 plugin 也只用它。flow 只经各模块的 `index.ts` 用它们；正式代码里没有为检验写的参数、环境变量、回放模式或假实现（CLAUDE.md"测试"）。这一节对应 flow 第 1–4 部分（任务书 12、14、16、17）：骨架、研究开始、第零轮、第 r 轮从开轮到收尾、一轮接一轮直到余额用完；余额为 0 时走到 `export` 停下，原因"导出还没写"（导出还没写）。轮次按"第 r 轮"写，不写死轮数。 +下面是 `src/flow/index.ts` 导出的全部内容，以后 plugin 也只用它。flow 只经各模块的 `index.ts` 用它们；正式代码里没有为检验写的参数、环境变量、回放模式或假实现（CLAUDE.md"测试"）。这一节对应 flow 第 1–5 部分（任务书 12、14、16、17、18）：骨架、研究开始、第零轮、第 r 轮从开轮（含诊断）到收尾、一轮接一轮直到余额用完；余额为 0 时走到 `export` 停下，原因"导出还没写"（导出还没写）。轮次按"第 r 轮"写，不写死轮数。 - `runStudy(input)`：推进一份研究，直到停下。输入三样，都必填：研究目录；配置（新研究交 `checkConfig` 检查过的配置，接着跑一份已有的研究写 null，以账本里冻结的那份为准；两者对不上就报错）；"停下"信号。每次启动走同一条路：打开账本；新研究先冻结配置，本机已有 `inforsi-<研究编号>` 的镜像就说明研究编号用过了，报错停下，要换一个编号（编号重复的新研究会把旧研究遗留的同名容器当成自己的运行）；已有研究的，核对冻结的配置、镜像还在、提示词哈希没变（开发入口开着时提示词可以变）；给开始了没结束的模型调用补写结束（出错、"进程中断"、墙钟时间测不到）；按容器名问 Docker 核对登记了没结果的正式运行（没启动的写"没启动"退还名额，启动了的补写确认启动、等它、判分、写结果，问不到的写"状态未知"事件并停下）；然后进主循环：读账本 → 问"下一步做什么" → 做这一步 → 回到开头。主循环里 `docker run` 没启动的，写完"没启动"结果（退还名额、记下 Docker 报错原文）就停下，停下原因带上报错原文，本进程不重跑；下次启动时照旧起新编号重跑。恢复时补写的"没启动"不停下。 - `StudyInput`（类型）：上面三样输入。 @@ -607,6 +630,12 @@ Jev 是一个判别器，和程序检查地位相同，不单独成一种记录 | `entryDraft`、`discussion1`、`discussion2`、`claim` | 这个阶段的关闭记录（`stageClose`）；阶段编号 `r0-<步骤>` | | `assignEntries` 分配 | 第 0 轮的入口分配决定（`decision`，种类 `entries`） | | `openRound`、`carryOverPool` | 不写记录：本轮名单和提案上限就是冻结的配置（方法.md 4.3 ①），留池候选由 materials 从记录算出（`Board.carried`） | +| `diagnosisProposal`、`diagnosisPrediction` 诊 1、诊 2 | 这一轮这个阶段的关闭记录；阶段编号 `r<轮>-diagnosisProposal`、`r<轮>-diagnosisPrediction`。余额只算本轮开始之前的运行，不到 2 次时本轮不诊断：诊 1–诊 4 不开阶段、不写任何记录，材料里的"本轮没有诊断：余额只剩……"从余额现算 | +| `diagnosisDedupe` 诊 2 判重合并 | 本轮的诊断实验判重合并决定（`decision`，种类 `diagnosisDedup`） | +| `selectDiagnosis` 诊 3 | 本轮的选诊断实验决定（`decision`，种类 `selectDiagnosis`）；定了不诊断也写，之后诊 4 的几步都不做 | +| `diagnosisImplementation` 诊 4 实现、核验、修复 | 最近一次核验里至少有一个实验通过（不通过的拿掉）；或修复次数用完、一个都不剩而且有"不跑"事件（`diagnosisNotRun`，`objects` 是本轮诊断）。阶段编号 `r<轮>-diagnosisImplementation-1`、`r<轮>-diagnosisVerification-1`、`r<轮>-diagnosisRepair-1`、`r<轮>-diagnosisVerification-2`……第几次从账本数，不另存 | +| `diagnosisRun` 诊 4 运行 | 本轮诊断运行最近一次有结果，而且不是"没启动"；没有诊断、或实验一个都没剩下时不登记 | +| `diagnosisSettle` 诊 4 结算 | 本轮的诊断结算记录（`diagnosisSettlement`，按轮次唯一）；没有诊断、或实验一个都没剩下时不写 | | `proposal`、`firstPrediction`、`critique`、`response`、`secondPrediction` | 这一轮这个阶段的关闭记录（`stageClose`）；阶段编号 `r<轮>-<阶段>` | | `dedupe` 判重合并 | 本轮的判重合并决定（`decision`，种类 `dedup`） | | `freezeProtocol` 冻结协议 | 本轮每个新候选都有冻结协议记录（`protocol`）；全部候选在同一个事务里写 | @@ -642,6 +671,10 @@ Jev 是一个判别器，和程序检查地位相同，不单独成一种记录 | 实现 | 每个入选候选一个实验员，从本批基线取工作目录，不同候选同时实现 | 候选 | | 核验 | 每个待核验候选一个核验员；首次核验本批全部，之后只核验刚修复的候选 | 候选 | | 修复 | 只给核验不通过的候选，交原实验员，从上次提交的代码接着改 | 候选 | +| 诊断解释与实验、诊断预测 | 每位研究者一个任务 | 无 | +| 诊断实现 | 一位实验员（`experimenter-<本轮诊断>`），把选中的全部实验写进一份诊断程序，从当前基线取工作目录，再放进只读的原始产物 | 本轮诊断（`r<轮>-diagnosis`） | +| 诊断核验 | 一位核验员（`verifier-<本轮诊断>`），逐个实验交结果；每次核验都覆盖全部选中的实验 | 本轮诊断 | +| 诊断修复 | 只在最近一次核验有实验不通过时，交原实验员，从上次提交的诊断程序接着改，只修不通过的实验；修复后再核验，次数用同一个 `maxRepairs` | 本轮诊断 | | 解释 | 每位研究者一个任务 | 无 | | 消融实现 | 每个选中的格一个实验员；第一次从被采纳的完整代码取工作目录，重做时从上一次交的这一格代码接着改；不同格同时实现 | 消融格 | | 消融核验 | 每个交了代码的格一个核验员；首次核验全部，之后只核验刚重做的格 | 消融格 | @@ -680,6 +713,18 @@ Jev 是一个判别器，和程序检查地位相同，不单独成一种记录 - 通过核验的候选用既有 `formalRun`，用途 `forward`；先登记占名额，确认启动才扣次，再独立判分记结果。**一批内一个接一个跑**，避免抢 CPU 改变按秒停下的程序成绩。没启动的处理同基线：本进程停下，下次启动起新编号。 - 本批全部有结果或“不跑”事件后，用 rules 的 `adopt`（内部用 `adoptionLine`），只传有效实测分数、本批基线分数和任务的方向、ε、已知最好；写种类 `adopt` 的决定，即使 `adopted` 为 null 也写。materials 据此公开本轮正向运行。 +**诊断**（来源：任务书 18 第 1.6 节，方法.md 4.3 诊 1–诊 4、6.12）： + +- 做不做：余额（`runBalance`，只算本轮开始之前的运行，用 `Board.runStatesBefore`）不到 2 次时，诊 1–诊 4 都不做。余额不算本轮诊断运行自己扣的次数，所以诊断这一段的后几步不会因为扣次后余额变小被误算成"不诊断"。诊断关着时这几步按配置跳过，和此前一致。 +- 诊 2 判重合并：本轮已公开的诊断实验按作者编号、第几个排好，两两问一次 Jev（jev 的 `judgeSameExperiment`），全部问完才写决定（种类 `diagnosisDedup`，规则 `jevSameExperiment` 版本 1，输入是每一对的 Jev 判断）。判为相同的连通成组（同判重合并），每组一个实验，编号 `d1`、`d2`……，组内第一个（作者编号最小、再按第几个）的正文就是合并后的正文。 +- 诊 3 选实验：输入照 rules 的 `DiagnosisRuleInput` 拼（余额取本轮开始之前，种子不用，写 `null`），规则取步骤清单里 `selectDiagnosis` 那一步的，调 `runDiagnosisRule`；定了不诊断也写一条决定。 +- 诊 4 实现、核验、修复：对象都是本轮诊断（`diagnosisId`）。实验员从当前基线取工作目录，tasks 的 `placeBaselineArtifact` 把产出当前基线那次运行的原始产物只读地放进去；存版本时用 tasks 的 `diagnosisUnversionedNames`（多跳过这个目录）。提交时相对当前基线比较、切块，Jev 对照的是选中实验的原文和全名指标；收下的提交和代码版本、位置表在同一事务写入。核验员逐个实验交结果，用 submissions 的 `diagnosisOutcome` 判；核验、修复的材料差异都是当前基线到诊断程序。 +- 拿掉实验：修复次数用完后最近一次核验仍不通过的实验拿掉，结算里记"证据不足：核验不通过"和原因。诊断程序的代码不动（核验员核过的就是这一份），运行时只调通过的实验的编号，每个实验分到总时限 ÷ 跑的实验数；拿掉的实验的代码留在程序里不被调用。判"通过"以最近一次核验为准，修复前通过过、修复后又不通过的实验也拿掉。一个都不剩时用 `diagnosisNotRunEvent` 写事件，不登记运行、不扣次、不结算。 +- 诊断运行：用 `formalRun` 同一个登记、启动、扣次入口（formal-run.ts 的 `diagnosisFormalRun`，用途 `diagnosis`，对象本轮诊断），一次运行一次扣次；没启动的处理同正向。运行记录的 `judged` 写 null，逐个实验的判分在结算前由 `judgeDiagnosisRun` 从运行目录做。诊断运行编号 `r<轮>-diagnosis-<第几次>`。 +- 诊断结算：逐个实验判分，再照 `SettleDiagnosisInput` 调 `settleDiagnosis` 写一条诊断结算（运行编号、逐个实验的判分、拿掉的实验和原因、输入输出）。每个合并后的实验的结局：跑了且有效分数是有效；跑了没有效分数、或核验不通过拿掉的是证据不足；没选中的是没做。诊断结算写下、诊断运行公开之后才开 ② 提案。 +- 恢复：和其他运行走同一条恢复路径。登记了还没结果的诊断运行，按容器名问 Docker：没启动的写"没启动"退还名额，已启动的补写确认启动（扣次）、等它、写结果（`judged` 为 null）；只核对、不重跑。结算前被打断，恢复后从运行目录重新判分。 +- 诊 1 的材料要产出当前基线那次运行的产物（`metrics.json`、`solution.json`、日志），flow 读运行目录传给 materials。 + **消融、收尾与研究结束**（来源：任务书 17 第 2.2、2.3 节，方法.md 4.3 ⑬–⑱、4.4、第七部分）： - 采纳了而且被采纳改法有 2 个以上因子才走消融；否则 ⑬–⑰ 不写记录（从采纳决定和冻结协议算得出），直接到收尾。 diff --git a/eval/diagnosis/research.json b/eval/diagnosis/research.json new file mode 100644 index 0000000..bcb22ce --- /dev/null +++ b/eval/diagnosis/research.json @@ -0,0 +1,34 @@ +{ + "studyId": "diag-eval", + "task": "/home/iomgaa/Projects/InfoRSI/tasks/circle_packing", + "models": { + "researchers": [ + { "provider": "iomgaa", "model": "glm-5.3", "thinking": "medium" }, + { "provider": "iomgaa", "model": "glm-5.3", "thinking": "medium" }, + { "provider": "iomgaa", "model": "glm-5.3", "thinking": "medium" } + ], + "experimenter": { "provider": "iomgaa", "model": "glm-5.3", "thinking": "medium" }, + "verifier": { "provider": "iomgaa", "model": "glm-5.3", "thinking": "medium" }, + "jev": "jev-latest" + }, + "proposalLimit": 1, + "batchLimit": 2, + "runs": 13, + "factorLimit": 3, + "maxRepairs": 1, + "lightChecks": { "maxCount": 2, "seconds": 5 }, + "promptDir": "/home/iomgaa/Projects/InfoRSI/prompts", + "devEntry": false, + "flow": { + "preset": "full", + "roundZero": { "enabled": true, "discussions": 2 }, + "firstPrediction": true, + "critique": true, + "diagnosis": { "enabled": true, "perResearcher": 2, "maxExperiments": 4 }, + "ablation": { "enabled": false }, + "batchRule": "bestExpected", + "openReserve": 0.1, + "knowledgeUse": 0.5, + "decay": 0.5 + } +} diff --git a/eval/round-zero/research.json b/eval/round-zero/research.json index 7b0881a..3badbe7 100644 --- a/eval/round-zero/research.json +++ b/eval/round-zero/research.json @@ -24,6 +24,7 @@ "roundZero": { "enabled": true, "discussions": 2 }, "firstPrediction": true, "critique": true, + "diagnosis": { "enabled": false }, "ablation": { "enabled": true, "cellRule": "valueMinusCost" }, "batchRule": "bestExpected", "openReserve": 0.1, diff --git a/src/config/schema.ts b/src/config/schema.ts index e7c65c3..328aea9 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -12,6 +12,7 @@ export const FIELD_CATEGORIES = ["用户配置", "方法参数"] as const; export type FieldCategory = (typeof FIELD_CATEGORIES)[number]; const CLOSED = { additionalProperties: false } as const; +const PositiveInteger = Type.Integer({ minimum: 1 }); /** 文字字段：不能为空，也不能只有空白。 */ const TEXT = { pattern: "\\S" }; /** @@ -68,7 +69,7 @@ const RoundZero = Type.Union([ /** 消融（方法.md 第十一部分"步骤开关"）：关，或开并写选格规则。 */ const Ablation = Type.Union([ Type.Object( - { enabled: method(Type.Literal(false), "消融关：不列消融格；解释照样写文字，但不给每格的三个数") }, + { enabled: method(Type.Literal(false), "消融关：不列消融格，也不写解释（解释就是对消融格的预测，方法.md 4.3 ⑫）") }, CLOSED, ), Type.Object( @@ -83,6 +84,19 @@ const Ablation = Type.Union([ ), ]); +/** 诊断（方法.md 4.3 诊 1–诊 4、第十一部分"步骤开关"）：关，或开并写两个方法参数（6.11）。 */ +const Diagnosis = Type.Union([ + Type.Object({ enabled: method(Type.Literal(false), "诊断关：每轮提案之前不做诊断实验") }, CLOSED), + Type.Object( + { + enabled: method(Type.Literal(true), "诊断开"), + perResearcher: method(PositiveInteger, "每人每轮诊断实验上限 A；现行值 2（方法.md 6.11）"), + maxExperiments: method(PositiveInteger, "一次诊断最多做几个实验 D；现行值 4（方法.md 6.11）"), + }, + CLOSED, + ), +]); + /** * 两种流程（方法.md 第十一部分）。完整流程写开关、规则和它才用得到的方法参数； * 朴素循环整套照搬，不能写任何开关、规则，也不写它用不到的 λ、u、衰减（它不预测、不消融）。 @@ -96,6 +110,7 @@ const FullFlow = Type.Object( Type.Boolean(), "挑刺与回应开不开（方法.md 4.3 ⑤ ⑥）；关掉时不挑刺、不回应，协议照样冻结；现行值 true", ), + diagnosis: method(Diagnosis, "诊断（方法.md 4.3 诊 1–诊 4）"), ablation: method(Ablation, "消融（方法.md 4.3 ⑬–⑰）"), batchRule: method( Type.Enum(BATCH_RULE_NAMES), @@ -121,14 +136,12 @@ const NaiveFlow = Type.Object( { preset: method( Type.Literal("naive"), - "朴素循环：方法.md 第十一部分的对照流程整套照搬，不挑刺、不回应、不预测、均匀随机选批、不消融", + "朴素循环：方法.md 第十一部分的对照流程整套照搬，不挑刺、不回应、不预测、均匀随机选批、不诊断、不消融", ), }, CLOSED, ); -const PositiveInteger = Type.Integer({ minimum: 1 }); - export const ResearchConfig = Type.Object( { studyId: user( diff --git a/src/config/steps.ts b/src/config/steps.ts index 02716da..d16e1a5 100644 --- a/src/config/steps.ts +++ b/src/config/steps.ts @@ -1,4 +1,11 @@ -import { BATCH_RULES, CELL_RULES, type BatchRuleName, type CellRuleName } from "../rules/index.ts"; +import { + BATCH_RULES, + CELL_RULES, + DIAGNOSIS_RULES, + type BatchRuleName, + type CellRuleName, + type DiagnosisRuleName, +} from "../rules/index.ts"; import type { ResearchConfig } from "./schema.ts"; // 步骤清单（docs/结构.md 第三部分，方法.md 第十一部分）：把配置里的流程、开关、规则翻译成这份研究要走的步骤。 @@ -23,6 +30,13 @@ export type StudyStepId = (typeof STUDY_STEPS)[number]; export const ROUND_STEPS = [ "openRound", // 1. 开轮：冻结本轮名单和提案上限 "carryOverPool", // 1. 开轮：处理留池候选 + "diagnosisProposal", // 诊 1. 诊断解释与诊断实验 + "diagnosisDedupe", // 诊 2. 判重合并 + "diagnosisPrediction", // 诊 2. 诊断预测 + "selectDiagnosis", // 诊 3. 选诊断实验（程序） + "diagnosisImplementation", // 诊 4. 实现、核验、修复 + "diagnosisRun", // 诊 4. 诊断运行 + "diagnosisSettle", // 诊 4. 诊断结算 "proposal", // 2. 提案 "dedupe", // 3. 判重合并 "firstPrediction", // 4. 第一次预测（看原始提案） @@ -50,11 +64,11 @@ export const ROUND_STEPS = [ export type RoundStepId = (typeof ROUND_STEPS)[number]; /** 决策点用哪条规则：规则名和版本，从 rules 的登记表取。 */ -export type RuleRef = { name: BatchRuleName | CellRuleName; version: number }; +export type RuleRef = { name: BatchRuleName | CellRuleName | DiagnosisRuleName; version: number }; type Status = { status: "execute" } | { status: "skip"; reason: string }; -/** 一步：执行，或按配置跳过（带原因）；只有选批、选格两个决策点带规则，其余为 null。 */ +/** 一步：执行，或按配置跳过（带原因）；只有选批、选格、选诊断实验三个决策点带规则，其余为 null。 */ export type Step = { id: S; rule: RuleRef | null } & Status; export interface StepList { @@ -82,7 +96,7 @@ export function stepList(config: ResearchConfig): StepList { const flow = config.flow; if (flow.preset === "naive") { const naive = skip("朴素循环不做这一步（方法.md 第十一部分）"); - return assemble("naive", (id) => (NAIVE_STEPS.has(id) ? EXECUTE : naive), batchRule("uniformRandom"), null); + return assemble("naive", (id) => (NAIVE_STEPS.has(id) ? EXECUTE : naive), batchRule("uniformRandom"), null, null); } // 完整流程：只有开关关掉的步骤跳过。 @@ -102,15 +116,27 @@ export function stepList(config: ResearchConfig): StepList { if (!flow.firstPrediction) off(["firstPrediction"], "配置关掉了第一次预测"); // 挑刺与回应关掉时，协议照样冻结（freezeProtocol 不跟着关）。 if (!flow.critique) off(["critique", "response"], "配置关掉了挑刺与回应"); - // 消融关掉时解释照样写文字，只是不给每格的三个数（方法.md 第十一部分"步骤开关"），所以 explanation 不跟着关。 + // 消融关掉时，解释也不做：解释就是对消融格的预测，⑬–⑰ 整段都不做（方法.md 4.3 ⑫）。 if (!flow.ablation.enabled) { off( - ["listCells", "selectCells", "ablationImplementation", "ablationVerification", "ablationRun", "settle"], + [ + "listCells", "explanation", "selectCells", "ablationImplementation", + "ablationVerification", "ablationRun", "settle", + ], "配置关掉了消融", ); } + if (!flow.diagnosis.enabled) { + off( + [ + "diagnosisProposal", "diagnosisDedupe", "diagnosisPrediction", "selectDiagnosis", + "diagnosisImplementation", "diagnosisRun", "diagnosisSettle", + ], + "配置关掉了诊断", + ); + } const cells = flow.ablation.enabled ? cellRule(flow.ablation.cellRule) : null; - return assemble("full", (id) => skipped[id] ?? EXECUTE, batchRule(flow.batchRule), cells); + return assemble("full", (id) => skipped[id] ?? EXECUTE, batchRule(flow.batchRule), cells, flow.diagnosis.enabled ? diagnosisRule() : null); } function batchRule(name: BatchRuleName): RuleRef { @@ -121,13 +147,20 @@ function cellRule(name: CellRuleName): RuleRef { return { name, version: CELL_RULES[name].version }; } +/** 选诊断实验只有一条规则：信息价值减成本（方法.md 6.12），配置里不选。 */ +function diagnosisRule(): RuleRef { + return { name: "valueMinusCost", version: DIAGNOSIS_RULES.valueMinusCost.version }; +} + function assemble( preset: StepList["preset"], status: (id: StudyStepId | RoundStepId) => Status, batch: RuleRef, cells: RuleRef | null, + diagnosis: RuleRef | null, ): StepList { - const ruleOf = (id: RoundStepId) => (id === "selectBatch" ? batch : id === "selectCells" ? cells : null); + const ruleOf = (id: RoundStepId) => + id === "selectBatch" ? batch : id === "selectCells" ? cells : id === "selectDiagnosis" ? diagnosis : null; return { preset, study: STUDY_STEPS.map((id) => ({ id, rule: null, ...status(id) })), diff --git a/src/flow/formal-run.ts b/src/flow/formal-run.ts index c1676b3..beb746e 100644 --- a/src/flow/formal-run.ts +++ b/src/flow/formal-run.ts @@ -1,7 +1,18 @@ import { rmSync } from "node:fs"; import type { Content } from "../ledger/index.ts"; -import { formalRun as runDefinition, judge, prepareJudging, prepareRun } from "../tasks/index.ts"; -import type { HypothesisMetric, JudgedResult, ProcessResult } from "../vocab/index.ts"; +import { + diagnosisRun as diagnosisDefinition, + formalRun as runDefinition, + judge, + judgeDiagnosis, + prepareDiagnosisJudging, + prepareDiagnosisRun, + prepareJudging, + prepareRun, + type DiagnosisExperiment, + type DiagnosisJudged, +} from "../tasks/index.ts"; +import type { HypothesisMetric, JudgedResult, ProcessResult, RunDefinition } from "../vocab/index.ts"; import { checkoutVersion } from "../workspace/index.ts"; import { containerName, freshDir, judgeContainerName, runPaths, workspaceRepo } from "./paths.ts"; import type { Study } from "./study.ts"; @@ -9,6 +20,8 @@ import type { Study } from "./study.ts"; // 共用动作二：跑一次正式运行（任务书 12 第 2.7 节）。先登记、再启动、再写结果，前后顺序由这个入口保证 // （方法.md 第七、九部分）：登记占名额，确认启动才扣次，确认没启动的写结果"没启动"、退还名额。 // 判分不扣次数，不登记。恢复时接着等一次已经启动的运行，走同一个 finishRun。 +// 诊断运行（一次运行跑几个实验，用途 diagnosis）走同一个登记、启动、等待的入口，只是准备目录和判分不同： +// 运行记录不带判分（judged 为 null），逐个实验的判分在诊断结算前由 judgeDiagnosisRun 从运行目录做。 export type RunPlan = Omit, "definition">; @@ -22,11 +35,30 @@ export type RunOutcome = "done" | "stopped" | { notStarted: string }; /** 跑一次正式运行：取出 version 的代码，在任务镜像里跑，判分，写结果。用户停下时不等容器，交回 "stopped"。 */ export async function formalRun(s: Study, plan: RunPlan, version: string, metrics: HypothesisMetric[]): Promise { const definition = runDefinition(s.task, s.frozen.image, String(s.frozen.seed)); + return launch(s, plan, definition, metrics, (paths) => { + checkoutVersion(workspaceRepo(s.dir), version, freshDir(paths.code)); + prepareRun(s.task, paths.code, freshDir(paths.run), metrics); + }); +} + +/** + * 跑一次诊断运行：一个容器、一次登记、一次扣次，里面按顺序跑 experiments 里的实验（总时限平分）。 + * version 是诊断程序，baselineRunDir 是产出当前基线的那次运行的运行目录（原始产物只读地交给每个实验）。 + */ +export async function diagnosisFormalRun(s: Study, plan: RunPlan, version: string, baselineRunDir: string, experiments: DiagnosisExperiment[]): Promise { + const definition = diagnosisDefinition(s.task, s.frozen.image, String(s.frozen.seed), experiments.map((e) => e.id)); + return launch(s, plan, definition, null, (paths) => { + checkoutVersion(workspaceRepo(s.dir), version, freshDir(paths.code)); + prepareDiagnosisRun(s.task, paths.code, baselineRunDir, freshDir(paths.run), experiments); + }); +} + +/** 先登记、准备运行目录、启动，确认启动才写"确认启动"；metrics 为 null 的是诊断运行（运行记录不带判分）。 */ +async function launch(s: Study, plan: RunPlan, definition: RunDefinition, metrics: HypothesisMetric[] | null, prepare: (paths: ReturnType) => void): Promise { s.ledger.append("runPlanned", { ...plan, definition }); const paths = runPaths(s.dir, plan.run); freshDir(paths.base); - checkoutVersion(workspaceRepo(s.dir), version, freshDir(paths.code)); - prepareRun(s.task, paths.code, freshDir(paths.run), metrics); + prepare(paths); const started = await s.runner.start(containerName(s.frozen.config.studyId, plan.run), definition, paths.run); if (!started.started) { s.ledger.append("runResult", { run: plan.run, process: { ending: "notStarted", reason: started.reason }, judged: null }); @@ -39,13 +71,14 @@ export async function formalRun(s: Study, plan: RunPlan, version: string, metric /** * 等一次已经确认启动的运行结束，判分，写结果，删掉容器。日志写进这次运行自己的日志目录。 * 收到"停下"信号就不等了，直接交回 "stopped"；容器不管它，下次启动时恢复入口再来接着等。 + * metrics 为 null 的是诊断运行：这里不判分，结果的 judged 写 null。 */ -export async function finishRun(s: Study, run: string, metrics: HypothesisMetric[]): Promise<"done" | "stopped"> { +export async function finishRun(s: Study, run: string, metrics: HypothesisMetric[] | null): Promise<"done" | "stopped"> { const name = containerName(s.frozen.config.studyId, run); const paths = runPaths(s.dir, run); const waited = await untilStopped(s.stop, s.runner.wait(name, emptyDir(paths.logs))); if (waited === "stopped") return "stopped"; - const judged = await judgeRun(s, run, waited, metrics); + const judged = metrics === null ? null : await judgeRun(s, run, waited, metrics); s.ledger.append("runResult", { run, process: waited, judged }); await s.runner.remove(name); return "done"; @@ -68,6 +101,23 @@ async function judgeRun(s: Study, run: string, formal: ProcessResult, metrics: H return judge(s.task, paths.run, paths.judge, formal, validator, metrics); } +/** + * 诊断运行的判分（诊断结算前做）：每个交出答案的实验单独判分。同 judgeRun：判分目录每次重新建空的， + * 判分容器还在就先删掉再用；恢复时从运行目录重新判，不存半截结果。experiments 是这次运行跑的那几个实验，次序同运行。 + */ +export async function judgeDiagnosisRun(s: Study, run: string, process: ProcessResult, experiments: DiagnosisExperiment[]): Promise { + const paths = runPaths(s.dir, run); + const step = prepareDiagnosisJudging(s.task, paths.run, emptyDir(paths.judge), s.frozen.image, process, experiments); + if (step.status === "judged") return step.result; + const name = judgeContainerName(s.frozen.config.studyId, run); + if ((await s.runner.status(name)) !== "absent") await s.runner.remove(name); + const started = await s.runner.start(name, step.definition, paths.judge); + if (!started.started) throw new Error(`诊断运行 ${run} 的校验器容器没有启动：${started.reason}`); + const validator = await s.runner.wait(name, emptyDir(paths.judgeLogs)); + await s.runner.remove(name); + return judgeDiagnosis(s.task, paths.run, paths.judge, process, validator, experiments); +} + /** 清空重建一个目录：恢复时会再等一次、再判一次，上一次没写完的日志和判分目录不留。 */ function emptyDir(dir: string): string { rmSync(dir, { recursive: true, force: true }); diff --git a/src/flow/loop.ts b/src/flow/loop.ts index 52a0aef..2f958b7 100644 --- a/src/flow/loop.ts +++ b/src/flow/loop.ts @@ -1,7 +1,7 @@ import { mkdirSync } from "node:fs"; import { resolve } from "node:path"; import type { ResearchConfig } from "../config/index.ts"; -import { forwardNotRunEvent, cellNotRunEvent } from "../materials/index.ts"; +import { cellNotRunEvent, diagnosisNotRunEvent, forwardNotRunEvent } from "../materials/index.ts"; import { Ledger } from "../ledger/index.ts"; import { Runner } from "../runner/index.ts"; import { fullConfig, readFrozen } from "./frozen.ts"; @@ -10,6 +10,7 @@ import { ledgerFile } from "./paths.ts"; import { recover } from "./recover.ts"; import { baselineRun, saveBaseline } from "./steps/baseline.ts"; import { ablationRun, selectCells, settle, unsealChosen } from "./steps/ablation.ts"; +import { diagnosisDedupe, diagnosisRun, diagnosisSettle, selectDiagnosisExperiments } from "./steps/diagnosis.ts"; import { forwardRun, adoptBatch } from "./steps/forward.ts"; import { dispatchStage } from "./steps/dispatch.ts"; import { dedupe, freezeProtocols, selectBatch } from "./steps/round.ts"; @@ -80,10 +81,12 @@ async function doStep(s: Study, next: Exclude): Pr return null; case "baselineRun": case "forwardRun": - case "ablationRun": { + case "ablationRun": + case "diagnosisRun": { const ran = next.do === "baselineRun" ? await baselineRun(s, next.run) : next.do === "forwardRun" ? await forwardRun(s, next.round, next.candidate, next.run) + : next.do === "diagnosisRun" ? await diagnosisRun(s, next.round, next.run) : await ablationRun(s, next.round, next.cell, next.run); if (ran === "stopped") return USER_STOPPED; if (ran === "done") return null; @@ -103,6 +106,18 @@ async function doStep(s: Study, next: Exclude): Pr case "assignEntries": assignEntries(s); return null; + case "diagnosisDedupe": + await diagnosisDedupe(s, next.round); + return null; + case "selectDiagnosis": + selectDiagnosisExperiments(s, next.round); + return null; + case "diagnosisNotRun": + s.ledger.append("event", diagnosisNotRunEvent(next.round, next.reason)); + return null; + case "diagnosisSettle": + await diagnosisSettle(s, next.round); + return null; case "dedupe": await dedupe(s, next.round); return null; diff --git a/src/flow/model-task.ts b/src/flow/model-task.ts index 9eda06f..fbbdd85 100644 --- a/src/flow/model-task.ts +++ b/src/flow/model-task.ts @@ -4,8 +4,9 @@ import { runSession, type SessionTask, type SessionEnding, type SubmitAttempt, t import { Board, buildMaterial, callId, queryMaterial, researcherId, type TaskMaterial } from "../materials/index.ts"; import { receive, type Incoming } from "../submissions/index.ts"; import { SUBMISSION_KINDS, type JevJudgment, type Json, type SubmissionKind } from "../vocab/index.ts"; +import { placeBaselineArtifact } from "../tasks/index.ts"; import { checkoutVersion } from "../workspace/index.ts"; -import { prepareSubmission, taskCode, lightChecker } from "./task-code.ts"; +import { prepareSubmission, taskArtifact, taskCode, lightChecker } from "./task-code.ts"; import { callPaths, freshDir, runPaths, workspaceRepo } from "./paths.ts"; import type { Study } from "./study.ts"; @@ -43,7 +44,7 @@ export async function runModelTask(s: Study, t: ModelT prompts: s.prompts, codeRoot: paths.code, changes, - artifacts: t.kind === "explanation" ? runArtifacts(s, t.stageId) : null, + artifacts: t.kind === "explanation" ? roundArtifacts(s, t.stageId) : t.kind === "diagnosisProposal" ? baselineArtifacts(s, t.stageId) : null, }); const model = role === "researcher" ? researcherModel(s, t.actor) : s.frozen.config.models[role]; // 先登记、后建目录：调用编号只在登记之后才算用掉，被杀在两者之间时下次的编号不会撞上旧目录。 @@ -60,6 +61,8 @@ export async function runModelTask(s: Study, t: ModelT for (const r of material.readable) checkoutVersion(workspaceRepo(s.dir), r.version, freshDir(r.dir)); const workDir = freshDir(paths.work); if (startVersion !== null) checkoutVersion(workspaceRepo(s.dir), startVersion, workDir); + const artifactFrom = taskArtifact(s, t); + if (artifactFrom !== null) placeBaselineArtifact(s.task, artifactFrom, workDir); let reply = 0; const common = { model, @@ -90,22 +93,34 @@ export async function runModelTask(s: Study, t: ModelT * 解释任务要的产物（materials 的 TaskRequest.artifacts）：本轮每次已公开正向运行的 metrics.json、solution.json * 和标准输出、标准错误日志（运行编号 → 文件名 → 文字）。完整代码那次的答案出自哪里，就靠这些产物判断。 */ -function runArtifacts(s: Study, stageId: string): Record> { +function roundArtifacts(s: Study, stageId: string): Record> { const board = new Board(s.ledger.records()); const round = board.stage(stageId).round; const artifacts: Record> = {}; for (const r of board.runs()) { if (r.plan.purpose !== "forward" || r.plan.round !== round || !board.runPublic(r)) continue; - const paths = runPaths(s.dir, r.plan.run); - const files: Record = {}; - for (const [dir, name] of [[paths.run, "metrics.json"], [paths.run, "solution.json"], [paths.logs, "stdout.log"], [paths.logs, "stderr.log"]] as const) { - if (existsSync(join(dir, name))) files[name] = readFileSync(join(dir, name), "utf8"); - } - artifacts[r.plan.run] = files; + artifacts[r.plan.run] = runFiles(s, r.plan.run); } return artifacts; } +/** 诊 1 要的产物：产出本轮当前基线的那次运行，同样的几个文件（运行编号 → 文件名 → 文字）。 */ +function baselineArtifacts(s: Study, stageId: string): Record> { + const board = new Board(s.ledger.records()); + const run = board.runOf(board.baselineOf(board.stage(stageId).round))!.plan.run; + return { [run]: runFiles(s, run) }; +} + +/** 一次运行留下的产物：metrics.json、solution.json 和两份日志，有哪个读哪个。 */ +function runFiles(s: Study, run: string): Record { + const paths = runPaths(s.dir, run); + const files: Record = {}; + for (const [dir, name] of [[paths.run, "metrics.json"], [paths.run, "solution.json"], [paths.logs, "stdout.log"], [paths.logs, "stderr.log"]] as const) { + if (existsSync(join(dir, name))) files[name] = readFileSync(join(dir, name), "utf8"); + } + return files; +} + /** 研究者用的模型：第 i 位研究者（researcherId(i)）用配置里第 i 个。 */ function researcherModel(s: Study, actor: string) { const models = s.frozen.config.models.researchers; diff --git a/src/flow/next-step.ts b/src/flow/next-step.ts index 696d05f..fcb8572 100644 --- a/src/flow/next-step.ts +++ b/src/flow/next-step.ts @@ -1,8 +1,8 @@ import type { RoundStepId, StudyStepId } from "../config/index.ts"; import type { Content, LedgerRecord } from "../ledger/index.ts"; -import { Board, CELL_NOT_RUN, cellId, DECISION_KINDS, experimenterId, factorKey, FORWARD_NOT_RUN, ORIGINAL_BASELINE, researcherId, splitCell, verifierId } from "../materials/index.ts"; +import { Board, CELL_NOT_RUN, cellId, DECISION_KINDS, DIAGNOSIS_NOT_RUN, diagnosisId, experimenterId, factorKey, FORWARD_NOT_RUN, ORIGINAL_BASELINE, researcherId, splitCell, verifierId } from "../materials/index.ts"; import { compareIds, listCells, runBalance, sharedSettings } from "../rules/index.ts"; -import { ablationOutcome, verificationOutcome } from "../submissions/index.ts"; +import { ablationOutcome, diagnosisOutcome, verificationOutcome } from "../submissions/index.ts"; import { SUBMISSION_KINDS, type Stage, type SubmissionKind } from "../vocab/index.ts"; import { readFrozen, type Frozen } from "./frozen.ts"; @@ -23,7 +23,7 @@ const ROUND_ZERO_STAGES = { type RoundZeroStep = keyof typeof ROUND_ZERO_STAGES; /** 第 r 轮的研究者阶段：步骤名就是阶段名（vocab STAGES）。 */ -type RoundStageStep = Extract; +type RoundStageStep = Extract; /** 阶段编号由 flow 起：r<轮>-<步骤>，一个步骤一个阶段。第 r 轮按配置跳过的步骤也用这个编号记"按配置跳过"。 */ export const stageIdOf = (round: number, step: string): string => `r${round}-${step}`; @@ -52,6 +52,11 @@ export type Next = | { do: "closeStage"; stageId: string } | { do: "dispatch"; stageId: string; tasks: TaskRef[] } | { do: "assignEntries" } + | { do: "diagnosisDedupe"; round: number } + | { do: "selectDiagnosis"; round: number } + | { do: "diagnosisNotRun"; round: number; reason: string } + | { do: "diagnosisRun"; round: number; run: string } + | { do: "diagnosisSettle"; round: number } | { do: "dedupe"; round: number } | { do: "freezeProtocols"; round: number } | { do: "selectBatch"; round: number } @@ -139,6 +144,28 @@ function roundStepNext(step: RoundStepId, round: number, view: View, frozen: Fro case "carryOverPool": // 不写记录：本轮名单和提案上限就是冻结的配置（方法.md 4.3 ①），留池候选由 materials 从记录算出（Board.carried）。 return null; + case "diagnosisProposal": + case "diagnosisPrediction": + // 诊 1、诊 2 → 这个阶段的关闭记录。余额不到 2 次（算到本轮开始前）本轮不诊断：不开阶段、不写记录。 + return diagnosing(view, round, frozen) ? stageNext(stageIdOf(round, step), step, round, (roster) => roundTasks(step, round, roster, view.board), view, frozen, researchers(frozen)) : null; + case "diagnosisDedupe": + // 诊 2 判重合并 → 本轮的诊断实验判重合并决定（decision，种类 diagnosisDedup）。 + if (!diagnosing(view, round, frozen)) return null; + return view.board.decision("diagnosisDedup", round) === undefined ? { do: "diagnosisDedupe", round } : null; + case "selectDiagnosis": + // 诊 3 → 本轮的选诊断实验决定（decision，种类 selectDiagnosis），定了不诊断也写。 + if (!diagnosing(view, round, frozen)) return null; + return view.board.decision("selectDiagnosis", round) === undefined ? { do: "selectDiagnosis", round } : null; + case "diagnosisImplementation": + // 诊 4 实现与核验 → 最近一次核验里至少有一个实验通过，或修复次数用完而一个都不剩、有"不跑"事件。 + return diagnosisCodeNext(round, view, frozen); + case "diagnosisRun": + // 诊 4 运行 → 本轮诊断运行最近一次有结果，而且不是"没启动"。 + return diagnosisProgramOf(view.board, round, frozen) === null ? null : diagnosisRunNext(round, view); + case "diagnosisSettle": + // 诊 4 结算 → 本轮的诊断结算记录（diagnosisSettlement，按轮次唯一）。 + if (diagnosisProgramOf(view.board, round, frozen) === null) return null; + return view.board.diagnosisSettlement(round) === undefined ? { do: "diagnosisSettle", round } : null; case "proposal": case "firstPrediction": case "critique": @@ -183,9 +210,6 @@ function roundStepNext(step: RoundStepId, round: number, view: View, frozen: Fro // 解释 → 这一轮解释阶段的关闭记录。 const adopted = view.board.decision("adopt", round)?.output.adopted ?? null; if (adopted === null || view.board.protocol(adopted).factors.length < 2) return null; - if (frozen.steps.round.some((s) => s.id === "listCells" && s.status === "skip")) { - return { do: "halt", reason: "配置关掉了消融、又采纳了多因子改法：不交三个数的解释还没写（cells 只按完整流程全开实现，做步骤开关时再补）", objects: [] }; - } return stageNext(stageIdOf(round, "explanation"), "explanation", round, (roster) => roster.map((actor) => ({ actor, kind: "explanation" as const, object: null })), view, frozen, researchers(frozen)); } case "selectCells": @@ -228,6 +252,7 @@ export function poolOf(board: Board, round: number): string[] { /** * 第 r 轮一个研究者阶段的任务（任务书 14 第 2.3 节）。同一个人的几个任务按这里的先后跑。 + * - 诊断解释与实验、诊断预测：每人一个任务，没有对象； * - 提案：每人一个任务，没有对象； * - 第一次预测：每人 × 本轮每个新候选（含自己的；留池候选不做，方法.md 4.3 ④）； * - 挑刺：每人 × 本轮每条不是自己写的原提案（监督者定，照参考项目）； @@ -238,6 +263,9 @@ function roundTasks(step: RoundStageStep, round: number, roster: readonly string const each = (kind: TaskRef["kind"], objects: readonly string[]): TaskRef[] => roster.flatMap((actor) => objects.map((object) => ({ actor, kind, object }))); switch (step) { + case "diagnosisProposal": + case "diagnosisPrediction": + return roster.map((actor) => ({ actor, kind: step, object: null })); case "proposal": return roster.map((actor) => ({ actor, kind: "proposal", object: null })); case "firstPrediction": @@ -318,7 +346,7 @@ const researchers = (frozen: Frozen): string[] => frozen.config.models.researche function assignedStage( step: string, round: number, - kind: "implementation" | "repair" | "verification" | "ablationImplementation" | "ablationVerification", + kind: "implementation" | "repair" | "verification" | "ablationImplementation" | "ablationVerification" | "diagnosisImplementation" | "diagnosisRepair" | "diagnosisVerification", candidates: readonly string[], view: View, frozen: Frozen, @@ -355,6 +383,82 @@ function verificationNext(round: number, view: View, frozen: Frozen): Next | nul return null; } +/** + * 第 round 轮诊不诊断的前提（方法.md 4.3 诊断一段）：余额至少 2 次，保证本轮至少还能跑 1 次正向。 + * 余额只算本轮开始前的运行：本轮诊断运行自己扣了次数以后，诊断这一段的后几步不会反过来算成"不诊断"。 + * 诊断关着的步骤清单把这几步都标了跳过，不会走到这里。 + */ +function diagnosing(view: View, round: number, frozen: Frozen): boolean { + return runBalance(frozen.config.runs, view.board.runStatesBefore(round)).balance >= 2; +} + +/** + * 诊 4 实现、核验与修复（方法.md 4.3 诊 4）：一位实验员把选中的全部实验写进一份诊断程序，一位核验员逐个实验核对。 + * 阶段编号按"第几次"排：诊断实现-1、核验-1，不通过的退回修复-1、再核验-2……第几次从已开关的阶段数出来，不另存； + * 每次核验都覆盖全部选中的实验。最近一次核验里还有实验不通过、修复次数又用完时，不通过的实验拿掉（diagnosisProgramOf）； + * 一个都不剩才写"不跑"事件（方法.md 4.3 诊 4）。 + */ +function diagnosisCodeNext(round: number, view: View, frozen: Frozen): Next | null { + if (chosenExperiments(view.board, round) === null) return null; + const object = diagnosisId(round); + for (let n = 1; ; n++) { + const work = n === 1 + ? assignedStage("diagnosisImplementation-1", round, "diagnosisImplementation", [object], view, frozen) + : assignedStage(`diagnosisRepair-${n - 1}`, round, "diagnosisRepair", [object], view, frozen); + if (work !== null) return work; + const verify = assignedStage(`diagnosisVerification-${n}`, round, "diagnosisVerification", [object], view, frozen); + if (verify !== null) return verify; + const verification = view.board.published("diagnosisVerification").filter((v) => v.round === round).at(-1)!; + if (diagnosisOutcome(verification.content).passed) return null; + if (n > frozen.config.maxRepairs) { + const { experiments, dropped } = diagnosisProgram(view.board, round, frozen.config.maxRepairs); + if (experiments.length > 0 || view.events(DIAGNOSIS_NOT_RUN).some((e) => e.objects.includes(object))) return null; + return { do: "diagnosisNotRun", round, reason: `选中的实验第 ${n} 次核验都不通过，修复次数上限 ${frozen.config.maxRepairs} 已用完：${dropped.map((d) => `${d.experiment}：${d.reason}`).join("；")}` }; + } + } +} + +/** 本轮诊断选中的实验（诊 3 的决定，按实验编号排）；本轮没有诊断（余额不到 2 次、定了不诊断）为 null。 */ +function chosenExperiments(board: Board, round: number): string[] | null { + const decision = board.decision("selectDiagnosis", round); + return decision === undefined || !decision.output.diagnose ? null : decision.output.chosen; +} + +/** + * 诊 4 核验做完之后本轮诊断实际要跑的实验：选中的实验里最近一次核验通过的；不通过的实验拿掉，各带原因 + * （记"证据不足：核验不通过"）。拿掉实验后诊断程序的代码不动：核验员核过的就是这一份，只是运行时不调不通过的实验的编号。 + * 只在诊断实现与核验这一步做完以后调用。 + */ +export function diagnosisProgram(board: Board, round: number, maxRepairs: number): { experiments: string[]; dropped: Array<{ experiment: string; reason: string }> } { + const verifications = board.published("diagnosisVerification").filter((s) => s.round === round); + const last = verifications.at(-1)!.content; + const { failing } = diagnosisOutcome(last); + return { + experiments: board.decision("selectDiagnosis", round)!.output.chosen.filter((e) => !failing.includes(e)), + dropped: failing.map((experiment) => { + const x = last.experiments.find((y) => y.experiment === experiment)!; + const what = `${x.followed ? "" : "没照写法做；"}${x.nothingElse ? "" : "动了写法以外的地方；"}${x.where}`; + return { experiment, reason: `第 ${verifications.length} 次核验仍不通过，修复次数上限 ${maxRepairs} 已用完：${what}` }; + }), + }; +} + +/** 本轮要跑的诊断实验：本轮没有诊断、或一个都没剩下为 null。 */ +export function diagnosisProgramOf(board: Board, round: number, frozen: Frozen): string[] | null { + if (chosenExperiments(board, round) === null) return null; + const { experiments } = diagnosisProgram(board, round, frozen.config.maxRepairs); + return experiments.length === 0 ? null : experiments; +} + +/** 诊 4 运行：和正向同一套，没启动的下次启动起新编号，已登记没结果的必须先由恢复入口核对。 */ +function diagnosisRunNext(round: number, view: View): Next | null { + const runs = view.board.runs().filter((r) => r.plan.purpose === "diagnosis" && r.plan.round === round); + const last = runs.at(-1); + if (last !== undefined && last.result === null) throw new Error(`诊断运行 ${last.plan.run} 登记了还没有结果：恢复入口应当已经核对过`); + if (last !== undefined && last.result!.process.ending !== "notStarted") return null; + return { do: "diagnosisRun", round, run: `r${round}-diagnosis-${runs.length + 1}` }; +} + /** ⑩：按选批顺序一次只交回一个运行；没启动的下次启动起新编号，不会把半批提前公开。 */ function forwardNext(round: number, view: View): Next | null { for (const candidate of view.board.decision("batch", round)!.output.chosen) { diff --git a/src/flow/recover.ts b/src/flow/recover.ts index 9936b8e..cab368f 100644 --- a/src/flow/recover.ts +++ b/src/flow/recover.ts @@ -43,9 +43,13 @@ export async function recover(s: Study): Promise<{ halt: string } | "stopped" | return null; } -/** 恢复和第一次运行共用原登记指标定义（引用指标也保留原来的缺失规则）；消融格缺失一律只记缺失（方法.md 第五部分"指标"）。 */ -function metricsOf(s: Study, plan: Content<"runPlanned">): HypothesisMetric[] { +/** + * 恢复和第一次运行共用原登记指标定义（引用指标也保留原来的缺失规则）；消融格缺失一律只记缺失（方法.md 第五部分"指标"）。 + * 诊断运行为 null：运行记录不带判分，逐个实验的判分在诊断结算前做。 + */ +function metricsOf(s: Study, plan: Content<"runPlanned">): HypothesisMetric[] | null { if (plan.purpose === "baseline") return []; + if (plan.purpose === "diagnosis") return null; const board = new Board(s.ledger.records()); if (plan.purpose === "forward") return protocolMetrics(board, board.protocol(plan.object!)); return ablationMetrics(board, splitCell(plan.object!).candidate); diff --git a/src/flow/steps/diagnosis.ts b/src/flow/steps/diagnosis.ts new file mode 100644 index 0000000..1d3c08f --- /dev/null +++ b/src/flow/steps/diagnosis.ts @@ -0,0 +1,165 @@ +import { judgeSameExperiment, type DedupExperiment } from "../../jev/index.ts"; +import { Board, diagnosisId, diagnosisKey, type DiagnosisDedupOutput } from "../../materials/index.ts"; +import { + compareIds, + runBalance, + runDiagnosisRule, + settleDiagnosis, + type DiagnosisExplanation, + type DiagnosisRuleInput, + type DiagnosisRuleName, + type ExperimentOutcome, + type SettleDiagnosisInput, +} from "../../rules/index.ts"; +import type { DiagnosisExperiment, DiagnosisJudged } from "../../tasks/index.ts"; +import type { Json } from "../../vocab/index.ts"; +import { diagnosisFormalRun, judgeDiagnosisRun, type RunOutcome } from "../formal-run.ts"; +import { diagnosisProgram, diagnosisProgramOf } from "../next-step.ts"; +import { baselineRunDir } from "../task-code.ts"; +import type { Study } from "../study.ts"; +import type { SamePair } from "./round.ts"; + +// 诊断这一段里程序自己做的四步（方法.md 4.3 诊 2–诊 4）：判重合并、选诊断实验、诊断运行、诊断结算。 +// 判重合并和选实验各写一条决定，结算写一条诊断结算，写下之前被打断就从账本重新算一遍；写下之后恢复时照原样用，不重算。 + +/** 一个作者交的一个诊断实验：标签 "<作者>.e<第几个>"，只在判重合并的输入里认实验用。 */ +type Item = { author: string; index: number; label: string; experiment: DedupExperiment }; + +/** + * 诊 2 判重合并（方法.md 4.3）：本轮已公开的诊断实验两两问一次 Jev，按作者编号、再按第几个排好顺序发， + * 全部问完才写决定：中途被打断就整轮重问，判重不扣运行次数。 + */ +export async function diagnosisDedupe(s: Study, round: number): Promise { + const board = new Board(s.ledger.records()); + const model = s.frozen.config.models.jev; + const items: Item[] = board + .published("diagnosisProposal") + .filter((p) => p.round === round) + .sort((a, b) => compareIds(a.author, b.author)) + .flatMap((p) => p.content.experiments.map((experiment, index) => ({ author: p.author, index, label: `${p.author}.e${index + 1}`, experiment }))); + const pairs: SamePair[] = []; + for (const [i, a] of items.entries()) { + for (const b of items.slice(i + 1)) pairs.push({ a: a.label, b: b.label, judgment: await judgeSameExperiment(model, a.experiment, b.experiment) }); + } + // SAFETY: JevJudgment 和 DiagnosisDedupOutput 都是纯 Json 值；账本在写入时校验 Json。 + s.ledger.append("decision", { + decision: "diagnosisDedup", + round, + rule: "jevSameExperiment", + ruleVersion: 1, + params: { jevModel: model }, + seed: null, + input: { pairs } as unknown as Json, + output: groupSameExperiment(items, pairs) as unknown as Json, + }); +} + +/** + * 判为相同的连成一组（同判重合并：A 和 B 相同、B 和 C 相同，A、B、C 就是一个实验）。每组合成一个实验， + * 按组里排最前的那一个排序，编号用 materials 的 diagnosisKey（d1…）；组内按作者编号、再按第几个排，第一个的正文就是合并后的正文。 + * items 要先按作者编号、第几个排好。 + */ +export function groupSameExperiment(items: readonly Pick[], pairs: readonly SamePair[]): DiagnosisDedupOutput { + const parent = new Map(items.map((x) => [x.label, x.label])); + const root = (x: string): string => { + while (parent.get(x) !== x) x = parent.get(x)!; + return x; + }; + for (const p of pairs) if (p.judgment.yes) parent.set(root(p.a), root(p.b)); + const groups = new Map>(); + for (const x of items) groups.set(root(x.label), [...(groups.get(root(x.label)) ?? []), { author: x.author, index: x.index }]); + return { experiments: [...groups.values()].map((sources, i) => ({ experiment: diagnosisKey(i), sources })) }; +} + +/** + * 诊 3 选诊断实验（方法.md 6.12）：基线、方向、λ、合并后的全部实验、每条解释对每个实验的三个数，加余额（本轮开始前）、 + * 每批上限、一次诊断最多几个实验 D、u、衰减，交给步骤清单里 selectDiagnosis 那一步的规则，写一条决定。 + * 规则没有随机，种子写 null；定了不诊断也写。 + */ +export function selectDiagnosisExperiments(s: Study, round: number): void { + const board = new Board(s.ledger.records()); + const { config, facts, steps } = s.frozen; + const { diagnosis } = config.flow; + if (!diagnosis.enabled) throw new Error("配置关掉了诊断，步骤清单里选诊断实验一步应当标了跳过"); + const input: DiagnosisRuleInput = { + baseline: board.baselineScore(board.baselineOf(round)), + direction: facts.finalMetric.direction, + lambda: config.flow.openReserve, + experiments: board.diagnosisExperiments(round).map((e) => e.id), + explanations: predictionsOf(board, round), + balance: runBalance(config.runs, board.runStatesBefore(round)).balance, + batchCap: config.batchLimit, + maxExperiments: diagnosis.maxExperiments, + u: config.flow.knowledgeUse, + decay: config.flow.decay, + }; + const rule = steps.round.find((x) => x.id === "selectDiagnosis")!.rule!.name as DiagnosisRuleName; + const { rule: used, ...output } = runDiagnosisRule(rule, input); + // SAFETY: rules 是纯计算，输入输出都是纯 Json 值；账本在写入时校验 Json。 + s.ledger.append("decision", { + decision: "selectDiagnosis", + round, + rule: used.name, + ruleVersion: used.version, + params: output.params, + seed: null, + input: input as unknown as Json, + output: output as unknown as Json, + }); +} + +/** 这一轮已冻结的各条诊断解释：作者的第 i 条（materials 的 diagnosisExplanationId），每条对每个合并后的实验给的三个数。选实验和结算共用。 */ +function predictionsOf(board: Board, round: number): DiagnosisExplanation[] { + const predictions = board.published("diagnosisPrediction").filter((p) => p.round === round); + return board.diagnosisExplanations(round).map((x) => ({ + id: x.id, + author: x.author, + experiments: predictions.find((p) => p.author === x.author)!.content.predictions.filter((y) => y.explanation === x.id).map((y) => ({ experiment: y.experiment, range: y.range })), + })); +} + +/** 要跑的实验的运行输入：编号加它登记的假设专用指标（全名），次序同 ids，也就是运行的次序。 */ +function experimentsOf(board: Board, round: number, ids: readonly string[]): DiagnosisExperiment[] { + const merged = board.diagnosisExperiments(round); + return ids.map((id) => ({ id, metrics: merged.find((e) => e.id === id)!.metrics })); +} + +/** + * 诊 4 诊断运行：用核验过的诊断程序，复用正式运行的登记、扣次入口，对象是本轮诊断。跑的只是核验里通过的实验： + * 不通过而拿掉的实验，代码留在程序里不动（核验员核过的就是这一份），运行时不调它的编号；每个实验分到总时限 ÷ 跑的实验数。 + */ +export function diagnosisRun(s: Study, round: number, run: string): Promise { + const board = new Board(s.ledger.records()); + const ran = diagnosisProgramOf(board, round, s.frozen)!; + const version = board.code(diagnosisId(round)).version; + return diagnosisFormalRun(s, { run, purpose: "diagnosis", round, object: diagnosisId(round) }, version, baselineRunDir(s, board, round), experimentsOf(board, round, ran)); +} + +/** + * 诊 4 结算（方法.md 4.3 诊 4、6.12）：先从运行目录给每个交出答案的实验单独判分，再照 SettleDiagnosisInput 拼输入， + * 写一条诊断结算。每个合并后的实验的结局：跑了且有效分数是有效；跑了没有效分数、或核验不通过拿掉的是证据不足；没选中的是没做。 + * 运行记录不带判分，所以被打断在判分和写结算之间时，恢复后从运行目录重新判。 + */ +export async function diagnosisSettle(s: Study, round: number): Promise { + const board = new Board(s.ledger.records()); + const { experiments: ran, dropped } = diagnosisProgram(board, round, s.frozen.config.maxRepairs); + const run = board.runs().filter((r) => r.plan.purpose === "diagnosis" && r.plan.round === round).at(-1)!; + const judged = await judgeDiagnosisRun(s, run.plan.run, run.result!.process, experimentsOf(board, round, ran)); + const input: SettleDiagnosisInput = { + baseline: board.baselineScore(board.baselineOf(round)), + direction: s.frozen.facts.finalMetric.direction, + epsilon: s.frozen.facts.epsilon, + lambda: s.frozen.config.flow.openReserve, + experiments: board.diagnosisExperiments(round).map((e) => ({ experiment: e.id, outcome: outcomeOf(e.id, ran, dropped.map((d) => d.experiment), judged) })), + explanations: predictionsOf(board, round), + }; + // SAFETY: 判分结果、结算的输入输出都是纯 Json 值；账本在写入时校验 Json。 + s.ledger.append("diagnosisSettlement", { round, result: { run: run.plan.run, judged, dropped, input, output: settleDiagnosis(input) } as unknown as Json }); +} + +function outcomeOf(experiment: string, ran: readonly string[], dropped: readonly string[], judged: DiagnosisJudged): ExperimentOutcome { + if (dropped.includes(experiment)) return { kind: "insufficient" }; + if (!ran.includes(experiment)) return { kind: "untested" }; + const j = judged.experiments.find((x) => x.id === experiment)!; + return j.verdict === "valid" && j.score.status === "measured" ? { kind: "valid", score: j.score.value } : { kind: "insufficient" }; +} diff --git a/src/flow/study.ts b/src/flow/study.ts index 95952d8..552729d 100644 --- a/src/flow/study.ts +++ b/src/flow/study.ts @@ -94,6 +94,7 @@ function studyFacts(frozen: Frozen, task: Task): StudyFacts { lambda: config.flow.openReserve, totalRuns: config.runs, proposalLimit: config.proposalLimit, + diagnosisExperimentLimit: config.flow.diagnosis.enabled ? config.flow.diagnosis.perResearcher : null, factorLimit: config.factorLimit, metrics: metricDefinitions(task), taskText: task.guide, diff --git a/src/flow/task-code.ts b/src/flow/task-code.ts index 6e6385d..7319cdd 100644 --- a/src/flow/task-code.ts +++ b/src/flow/task-code.ts @@ -1,19 +1,20 @@ import { join } from "node:path"; import type { Content } from "../ledger/index.ts"; import { codeHunks } from "../jev/index.ts"; -import { Board, protocolMetrics, splitCell, type CodeChanges } from "../materials/index.ts"; +import { Board, diagnosisId, protocolMetrics, splitCell, type CodeChanges } from "../materials/index.ts"; import type { KnownOf } from "../submissions/index.ts"; -import { lightCheck, protectedChanges, readLightCheck } from "../tasks/index.ts"; +import { diagnosisUnversionedNames, lightCheck, protectedChanges, readLightCheck } from "../tasks/index.ts"; import type { Json, Submission, SubmissionKind } from "../vocab/index.ts"; import { compareVersions, saveVersion } from "../workspace/index.ts"; import type { TaskRef } from "./next-step.ts"; -import { callPaths, containerName, freshDir, workspaceRepo } from "./paths.ts"; +import { callPaths, containerName, freshDir, runPaths, workspaceRepo } from "./paths.ts"; import type { Study } from "./study.ts"; /** * 实现取本批基线；修复取上次提交。核验、修复的材料差异始终相对本批基线。 * 消融实现第一次取被采纳的完整代码，重做取上一次交的这一格代码（materials 的消融实现材料就是这样写的）； * 消融核验的材料差异是完整代码到这一格代码（方法.md 4.3 ⑯）。 + * 诊断实现取当前基线；诊断修复取上次提交的诊断程序；诊断核验、修复的材料差异是当前基线到诊断程序（方法.md 4.3 诊 4）。 */ export function taskCode(s: Study, t: TaskRef & { stageId: string }): { changes: CodeChanges | null; startVersion: string | null } { const board = new Board(s.ledger.records()); @@ -26,6 +27,13 @@ export function taskCode(s: Study, t: TaskRef & { stageId: string }): { changes: const { candidate } = splitCell(t.object!); return { changes: compareVersions(workspaceRepo(s.dir), board.code(candidate).version, board.code(t.object!).version), startVersion: null }; } + if (t.kind === "diagnosisImplementation" || t.kind === "diagnosisRepair" || t.kind === "diagnosisVerification") { + const round = board.stage(t.stageId).round; + const baseline = board.code(board.baselineOf(round)).version; + if (t.kind === "diagnosisImplementation") return { changes: null, startVersion: baseline }; + const program = board.code(t.object!).version; + return { changes: compareVersions(workspaceRepo(s.dir), baseline, program), startVersion: t.kind === "diagnosisRepair" ? program : null }; + } if (t.kind !== "implementation" && t.kind !== "repair" && t.kind !== "verification") return { changes: null, startVersion: null }; const baseline = board.code(board.baselineOf(board.stage(t.stageId).round)).version; if (t.kind === "implementation") return { changes: null, startVersion: baseline }; @@ -33,6 +41,18 @@ export function taskCode(s: Study, t: TaskRef & { stageId: string }): { changes: return { changes: compareVersions(workspaceRepo(s.dir), baseline, version), startVersion: t.kind === "repair" ? version : null }; } +/** 产出第 round 轮当前基线的那次运行的运行目录：诊断实验要接着算的原始产物就在里面。 */ +export function baselineRunDir(s: Study, board: Board, round: number): string { + return runPaths(s.dir, board.runOf(board.baselineOf(round))!.plan.run).run; +} + +/** 诊断实现、修复的工作目录要放进的原始产物所在的运行目录（取出代码之后，tasks 的 placeBaselineArtifact 放）；别的任务为 null。 */ +export function taskArtifact(s: Study, t: TaskRef & { stageId: string }): string | null { + if (t.kind !== "diagnosisImplementation" && t.kind !== "diagnosisRepair") return null; + const board = new Board(s.ledger.records()); + return baselineRunDir(s, board, board.stage(t.stageId).round); +} + /** * 提交时才知道的东西：提案份数、回应原提案，或实验员当前工作目录的快照和相对本批基线的改动。 * 快照写入 workspace 后才问 Jev；收下才把 code 和 submission 在同一个事务写进账本。 @@ -61,6 +81,24 @@ export function prepareSubmission( code: { object: t.object!, version: saved.version, locations: null }, }; } + if (t.kind === "diagnosisImplementation" || t.kind === "diagnosisRepair") { + // 诊断程序：存的名单多跳过宿主放进来的原始产物目录；Jev 对照的是选中实验的原文和要采集的指标。 + const saved = saveVersion(workspaceRepo(s.dir), workDir, diagnosisUnversionedNames(s.task)); + if (!saved.saved) return { known: { code: saved }, code: null }; + const round = board.stage(t.stageId).round; + const changes = compareVersions(workspaceRepo(s.dir), board.code(board.baselineOf(round)).version, saved.version); + const chosen = board.diagnosisExperiments(round).filter((e) => board.decision("selectDiagnosis", round)!.output.chosen.includes(e.id)); + const impl = content as Submission<"diagnosisImplementation">; + return { + known: { code: { + saved: true, + protectedFiles: protectedChanges(s.task, changes.files.map((f) => f.path)), + hunks: codeHunks(changes), + protocol: { body: chosen.map((e) => `### ${e.id} ${e.name}\n\n动哪一处：${e.spot}\n\n${e.body}`).join("\n\n"), metrics: chosen.flatMap((e) => e.metrics) }, + } }, + code: { object: t.object!, version: saved.version, locations: impl.locations as Json }, + }; + } if (t.kind !== "implementation" && t.kind !== "repair") return { known: null, code: null }; const saved = saveVersion(workspaceRepo(s.dir), workDir, s.frozen.facts.unversionedNames); if (!saved.saved) return { known: { code: saved }, code: null }; diff --git a/src/jev/index.ts b/src/jev/index.ts index 54e9db9..c514bc5 100644 --- a/src/jev/index.ts +++ b/src/jev/index.ts @@ -1,11 +1,15 @@ // jev：问 Jev 的问法和阈值（docs/结构.md 第二部分）。判断的类型 JevJudgment 在 vocab。 export { + checkExperiment, checkHunk, checkProtocol, codeHunks, judgeSameChange, + judgeSameExperiment, type CodeHunk, + type DedupExperiment, type DedupFactor, + type DiagnosisExperiment, type FrozenProtocol, type JevCheck, type ProtocolMetric, diff --git a/src/jev/jev.ts b/src/jev/jev.ts index 086b816..6bd08eb 100644 --- a/src/jev/jev.ts +++ b/src/jev/jev.ts @@ -1,9 +1,10 @@ import type { HypothesisMetric, JevJudgment, Json, Submission } from "../vocab/index.ts"; import { questionsOf, taskOf, type Question } from "./questions.ts"; -// 问 Jev：从带类型的输入拼材料、发请求、读回答（docs/方法.md 4.3 ③⑥⑨）。 +// 问 Jev：从带类型的输入拼材料、发请求、读回答（docs/方法.md 4.3 ③⑥⑨ 和诊 1、诊 2）。 // 材料里放什么、字段叫什么只在这里定。分请求照参考项目 src/judge/judge.ts： // 判重一对一次请求；冻结条款三问每问单独一次请求，材料相同；代码改动一个差异块一次请求，五问同在。 +// 诊断实验的判重、冻结检查照判重、冻结条款的写法；诊断程序的代码改动直接用 checkHunk。 type Factor = Submission<"proposal">["factors"][number]; @@ -13,6 +14,12 @@ type Factor = Submission<"proposal">["factors"][number]; */ export type DedupFactor = Pick; +/** 判重看的诊断实验：名称、动哪一处、怎么动。能分开哪几条解释写在正文里，不影响"是不是同一个实验"。 */ +export type DedupExperiment = Pick["experiments"][number], "name" | "spot" | "body">; + +/** 冻结检查看的诊断实验：名称、动哪一处、怎么动，加登记的假设专用指标（编号是短名）。 */ +export type DiagnosisExperiment = DedupExperiment & Pick["experiments"][number], "metrics">; + /** 一份冻结协议的条款（docs/方法.md 4.3 ⑥"冻结协议包括"），字段与统一提交相同。 */ export type FrozenProtocol = Pick, "factors" | "safeguards" | "metrics" | "body" | "clauses">; @@ -49,6 +56,31 @@ export async function judgeSameChange(model: string, a: readonly DedupFactor[], return judgment as JevJudgment; } +/** 诊 2 判重：两个诊断实验是不是同一个实验。 */ +export async function judgeSameExperiment(model: string, a: DedupExperiment, b: DedupExperiment): Promise { + const state = { + task: taskOf("diagnosisDedup"), + experiment_a: dedupExperiment(a), + experiment_b: dedupExperiment(b), + }; + const [judgment] = await ask(model, state, questionsOf("diagnosisDedup")); + return judgment as JevJudgment; +} + +/** 诊 1 查一个诊断实验：改动了评测、自相矛盾、写得太含糊。三问逐个发，材料相同。 */ +export async function checkExperiment(model: string, experiment: DiagnosisExperiment): Promise { + const state = { + task: taskOf("diagnosisFreeze"), + experiment: { + ...dedupExperiment(experiment), + metrics: experiment.metrics.map((m) => ({ id: m.id, howMeasured: m.howMeasured, missing: m.missing })), + }, + }; + const judgments: JevJudgment[] = []; + for (const question of questionsOf("diagnosisFreeze")) judgments.push(...(await ask(model, state, [question]))); + return check(judgments); +} + /** ⑥ 查一份冻结协议的条款：改动了评测、自相矛盾、写得太含糊。三问逐个发，材料相同。 */ export async function checkProtocol(model: string, protocol: FrozenProtocol): Promise { const state = { @@ -83,6 +115,10 @@ function dedupFactor(f: DedupFactor): { [key: string]: Json } { return { name: f.name, method: f.method, keySettings: f.keySettings }; } +function dedupExperiment(e: DedupExperiment): { [key: string]: Json } { + return { name: e.name, spot: e.spot, body: e.body }; +} + function check(judgments: JevJudgment[]): JevCheck { return { flagged: judgments.some((j) => j.yes), judgments }; } diff --git a/src/jev/questions.ts b/src/jev/questions.ts index 2e9f26a..741a31f 100644 --- a/src/jev/questions.ts +++ b/src/jev/questions.ts @@ -1,13 +1,17 @@ import { hash, type Json } from "../vocab/index.ts"; -// Jev 的九问（docs/方法.md 4.3 ③⑥⑨）：每一处的材料说明、问题原文、阈值。 +// Jev 的十三问（docs/方法.md 4.3 ③⑥⑨ 和诊 1、诊 2）：每一处的材料说明、问题原文、阈值。 // 措辞、判"是"判"否"的说明和示例句取自参考项目 src/judge/questions.ts（填圆那一套）， // 只改了必须改的：点 → 因子，字段名对上 vocab 的提案和冻结协议，改动对上 workspace 的比较结果。 // 写法照参考项目 research-wiki/参考-Jev使用指南.md：问题用英文、一问只问一件事、 // 统一问坏的一面（概率高 = 有问题或"是同一个"）、判据带 not_for 和例子、问题里用反引号点到材料路径。 -/** 用到 Jev 的三处：判重、冻结条款、一段代码改动。 */ -export type Site = "dedup" | "freeze" | "diff"; +/** + * 用到 Jev 的五处：判重、冻结条款、一段代码改动，以及诊断实验的判重和冻结检查。 + * 诊断的两处材料是"一个诊断实验"（动当前基线的一处），不是候选提案和冻结协议，所以各有自己的材料说明和问法 + * （方法.md 4.3 诊 1、诊 2）；一段代码改动的五问直接沿用 diff（诊 4）。 + */ +export type Site = "dedup" | "freeze" | "diff" | "diagnosisDedup" | "diagnosisFreeze"; /** 一条是非题（Jev 的 noul）。 */ export interface Noul { @@ -24,6 +28,9 @@ const THRESHOLDS: Readonly>>> = { dedup: { same_change: 0.3 }, freeze: { self_contradictory: 0.5, changes_measurement: 0.21, too_vague: 0.78 }, diff: { unseeded_randomness: 0.55, reads_clock: 0.55, touches_measurement: 0.65, early_exit: 0.63, zero_for_unmeasured: 0.62 }, + // 诊断实验的两处阈值暂用 dedup、freeze 的值，没有在诊断实验的材料上校准过，等真实数据（任务书 18 第三块）再校准。 + diagnosisDedup: { same_experiment: 0.3 }, + diagnosisFreeze: { self_contradictory: 0.5, changes_measurement: 0.21, too_vague: 0.78 }, }; /** 放进材料 `task` 字段的说明：这是什么材料、各部分是什么。 */ @@ -44,6 +51,18 @@ const TASKS: Readonly> = { "`protocol.metrics` lists the metrics the protocol requires the implementer " + "to compute in the candidate code and write to metrics.json (each with its `howMeasured` and `missing` rule; an empty list means none). " + "Only this hunk is shown; judge only what the hunk itself adds or changes.", + diagnosisDedup: + "Two diagnosis experiments from the same round of a study that improves a program by changing its code. " + + "A diagnosis experiment takes the current baseline program, changes ONE spot of it (`spot`) as `body` describes " + + "(replace a step with a clearly stronger version, replace it with another method, remove it, or relax a restriction), " + + "runs the changed program once and scores the answer it returns. " + + "The experiments were written by different researchers, in Chinese or English.", + diagnosisFreeze: + "One diagnosis experiment in a study that improves a program by changing its code. " + + "An implementer will change the baseline program at `experiment.spot` exactly as `experiment.body` says and nothing else; " + + "the unchanged task harness will then run the changed program once, and the task's validator will score the answer it returns. " + + "The implementer may not edit the task's harness or validator files. " + + "`experiment.metrics` lists extra diagnostic metrics the changed program must record (an empty list means none).", }; const QUESTIONS: Readonly>>> = { @@ -292,6 +311,120 @@ const QUESTIONS: Readonly>>> = { }, }, }, + // 诊 2 诊断实验判重：一对实验一次请求，写法照 dedup。 + diagnosisDedup: { + same_experiment: { + type: "noul", + instructions: { + question: + "Do `experiment_a` and `experiment_b` change the same spot of the baseline in the same way, so that one run would answer both?", + focus: + "Compare where each experiment changes the baseline (`spot`) and how (`body`): the kind of change (stronger version, other method, removal, relaxed restriction) and its settings. " + + "Ignore wording, language, level of detail, the experiment's name, and which explanation it says it can separate from which. " + + "The same spot changed in a different way, or the same change with different settings or sizes, is NOT the same experiment: each would need its own run.", + }, + criteria: { + true: { + what: "Both change the same spot of the baseline in the same way with the same settings; they differ only in wording, language or detail", + examples: [ + "'replace the optimizer step with an exact solver, no time limit' vs 'swap the optimizer for an exact solve without any time cap'", + "'remove the final repair pass' vs 'delete the repair pass at the end of the pipeline'", + ], + }, + false: { + what: "They change different spots, change the same spot in a different way, or use the same change with different settings", + not_for: "Two experiments that merely test the same explanation, or merely touch the same function at different spots", + examples: [ + "'remove the repair pass' vs 'replace the repair pass with an exact solver'", + "'run the search for 10x more steps' vs 'run the search for 100x more steps'", + "'relax the iteration limit of the search' vs 'relax the time limit of the whole program'", + ], + }, + }, + }, + }, + + // 诊 1 诊断实验冻结检查：每问单独一次请求，材料相同；问法照 freeze，材料换成"一个诊断实验"。 + diagnosisFreeze: { + self_contradictory: { + type: "noul", + instructions: { + question: + "Does `experiment` ask for something that cannot be done as written: do two of its statements exclude each other, or does `experiment.body` change places other than the one `experiment.spot` names?", + focus: + "Take the change described in `experiment.body` and check it against `experiment.spot`: the experiment must change that one spot and keep everything else of the baseline as it is. " + + "A change that cannot be carried out without also changing something else, or statements that require things that cannot both hold, make it contradictory. " + + "A statement that is merely unusual, ambitious or likely to give a worse score is not a contradiction.", + }, + criteria: { + true: { + what: "Carrying out the experiment necessarily changes more than the named spot, or two statements require things that cannot both hold", + examples: [ + "`spot` names the radius computation while `body` also asks to replace the starting layout", + "`body` says remove the step and also tune its step size", + "`body` says change only the solver and also keep the solver's code unchanged", + ], + }, + false: { + what: "All statements can hold at the same time in one implementation that changes only the named spot", + not_for: "Experiments that may not improve the score, or statements that are vague but not conflicting", + examples: ["replace the solver with an exact LP and keep everything else as in the baseline", "remove the final repair pass"], + }, + }, + }, + changes_measurement: { + type: "noul", + instructions: { + question: + "Does `experiment` require changing how the answer is scored or checked, rather than changing the solution that gets scored?", + focus: + "The answer of the changed program is scored by the task's validator and harness. " + + "Changing the algorithm that produces the answer is allowed, including relaxing a limit or a check inside the program itself, as long as the returned answer is still scored as before. " + + "Changing how the returned value is summed, scored, validated, rounded or reported, or the validator or harness themselves, is a change of measurement. " + + "Rules for the experiment's own diagnostic metrics (what they count, how they are aggregated) do not change the score.", + }, + criteria: { + true: { + what: "The experiment asks to alter the scoring or validation: the formula of the score, what gets counted, tolerances, the harness or validator, or the value reported instead of the real one", + examples: [ + "report the sum of diameters instead of the sum of radii", + "count only items above a size threshold when summing", + "raise the overlap tolerance in the validator", + ], + }, + false: { + what: "The experiment only changes how the answer is built or searched; the score is computed from its output as before", + not_for: "Adding diagnostic metrics that are recorded next to the score without replacing it", + examples: ["replace the greedy step with an exact solve", "remove the repair pass and see what the raw answer scores"], + }, + }, + }, + too_vague: { + type: "noul", + instructions: { + question: + "Does `experiment` leave the implementer to guess what to change, for example which spot of the baseline to edit, what to replace it with, or the settings of the stronger version?", + focus: + "Read `experiment.spot` and `experiment.body` as an implementer who must follow them without asking questions. " + + "Brief text is fine when it names the spot, the kind of change and its settings. " + + "Judge only whether each statement is specific; whether two statements conflict is checked separately.", + }, + criteria: { + true: { + what: "An implementer would have to invent a key part of the experiment: the spot to change, what to put there, or its main settings", + examples: ["make the search better somewhere", "replace the optimizer with a stronger one", "relax some of the limits"], + }, + false: { + what: "The spot, the kind of change and its settings are stated concretely enough to implement directly", + not_for: "Experiments that leave only minor coding details, such as variable names, to the implementer", + examples: [ + "in the radius step, replace the heuristic with an exact LP solved by scipy linprog; no time limit", + "remove the final repair pass and return the raw answer", + ], + }, + }, + }, + }, }; /** 一问的全部内容：问题原文、阈值，以及材料说明、问题原文、阈值三者合起来的哈希。 */ diff --git a/src/ledger/kinds.ts b/src/ledger/kinds.ts index 39da0f0..f28c355 100644 --- a/src/ledger/kinds.ts +++ b/src/ledger/kinds.ts @@ -17,7 +17,7 @@ import { type SubmissionKind, } from "../vocab/index.ts"; -// 账本记录的种类（docs/结构.md 第四部分的 11 种记录，细分成下面 19 种），每种一份内容定义（TypeBox）。 +// 账本记录的种类（docs/结构.md 第四部分的 11 种记录，细分成下面 20 种），每种一份内容定义（TypeBox）。 // 下面写"只有一条""只一次"的，由 schema.ts 的唯一约束保证。 // 配置、冻结协议、决定、结算的内容由别的模块产生，这里只检查最少的标识字段，其余只检查是 Json。 @@ -131,7 +131,7 @@ export const KINDS = { submission: Type.Unsafe({}), /** * 3 退回：被退回的那份内容原样存（它可能正是因为不合格式才被退回），连同退回理由。 - * target 是这次任务的对象，与提交的 target 同义（候选、原提案、消融格，没有对象的写 null）： + * target 是这次任务的对象，与提交的 target 同义（候选、原提案、消融格、本轮诊断，没有对象的写 null）： * 同一人在同一阶段可能对几个对象各做一次同种任务，重新发起时按它找本人这个对象的上一次退回。 * jev 是这次退回用到的 Jev 判断（见 JevUsed），判"否"的几问也一起存；没用到 Jev 的写 null。 */ @@ -161,11 +161,11 @@ export const KINDS = { /** 6 冻结代码：候选或消融格的代码版本号（由 workspace 给出）和因子位置表。位置表先存 Json。 */ code: Type.Object({ object: Id, version: Text, locations: AnyJson }, CLOSED), - /** 7 运行·登记要跑（占名额）：运行编号由 flow 起，一个编号只登记一次；用途、轮次、对象（候选或格；基线没有）、运行定义。 */ + /** 7 运行·登记要跑（占名额）：运行编号由 flow 起，一个编号只登记一次；用途、轮次、对象（候选、格或本轮诊断；基线没有）、运行定义。 */ runPlanned: Type.Object( { run: Id, - purpose: Type.Enum(["baseline", "forward", "ablation"], { type: "string" }), + purpose: Type.Enum(["baseline", "forward", "ablation", "diagnosis"], { type: "string" }), round: Round, object: OptionalId, definition: RunDefinition, @@ -178,7 +178,7 @@ export const KINDS = { runResult: Type.Object({ run: Id, process: ProcessResult, judged: Type.Union([JudgedResult, Type.Null()]) }, CLOSED), /** - * 8 决定：选批、采纳、选格、押的因子、入口分配、判重合并。decision 是决定种类。 + * 8 决定：选诊断实验、选批、采纳、选格、押的因子、入口分配、判重合并（提案和诊断实验）。decision 是决定种类。 * 输入输出存 Json：ledger 不认识 rules。判重合并的输入里带 Jev 的原始概率。 */ decision: Type.Object( @@ -197,6 +197,8 @@ export const KINDS = { /** 9 结算：每轮最多采纳一个改法，所以最多一次消融，结算按轮次一轮一条。各格结局、配合关系、解释的确认或否定等先存 Json。 */ settlement: Type.Object({ round: Round, result: AnyJson }, CLOSED), + /** 9 诊断结算：每轮最多一次诊断，结算也按轮次一轮一条（方法.md 6.12）。每个实验的结局与实测、解释的确认或否定等先存 Json。 */ + diagnosisSettlement: Type.Object({ round: Round, result: AnyJson }, CLOSED), /** 10 模型调用开始：先登记再调用（方法.md 第九部分）。附这次任务的全部材料，用来重现模型当时看到了什么。一次调用只有一条。 */ callStarted: Type.Object({ call: Id, actor: Id, stageId: OptionalId, model: Text, materials: AnyJson }, CLOSED), diff --git a/src/ledger/ledger.ts b/src/ledger/ledger.ts index 375e902..85620c1 100644 --- a/src/ledger/ledger.ts +++ b/src/ledger/ledger.ts @@ -83,7 +83,7 @@ export class Ledger { } /** - * 追加一条记录，返回存下的那一条。先按种类检查内容，再做写入时的四种拒收检查（rejections.ts）。 + * 追加一条记录，返回存下的那一条。先按种类检查内容，再做写入时的五种拒收检查（rejections.ts）。 * 写提案时，作者自评从提案里拆出来，在同一个事务里另存成一条封存，返回的是提案那一条。 */ append(kind: K, content: Input): LedgerRecord { @@ -117,16 +117,24 @@ export class Ledger { this.#first("stageOpen", "json_extract(content, '$.stageId') = ?", stageId) as LedgerRecord<"stageOpen"> | undefined, latest: (slot) => this.latestSubmission(slot), runStarted: (run) => this.#first("runStarted", "json_extract(content, '$.run') = ?", run) !== undefined, - ablationStarted: (round) => + ablationStarted: (round) => this.#purposeStarted("ablation", round), + diagnosisStarted: (round) => this.#purposeStarted("diagnosis", round), + }; + + /** 这一轮有没有某种用途的运行已经确认启动。 */ + #purposeStarted(purpose: "ablation" | "diagnosis", round: number): boolean { + return ( this.#one( `SELECT s.seq FROM records s JOIN records p ON p.kind = 'runPlanned' AND json_extract(p.content, '$.run') = json_extract(s.content, '$.run') WHERE s.kind = 'runStarted' - AND json_extract(p.content, '$.purpose') = 'ablation' + AND json_extract(p.content, '$.purpose') = ? AND json_extract(p.content, '$.round') = ?`, + purpose, round, - ) !== undefined, - }; + ) !== undefined + ); + } // ------------------------------------------------------------------------- // 读取 diff --git a/src/ledger/rejections.ts b/src/ledger/rejections.ts index 21ad79c..1914c87 100644 --- a/src/ledger/rejections.ts +++ b/src/ledger/rejections.ts @@ -1,8 +1,8 @@ import { SUBMISSION_KINDS, type Stage } from "../vocab/index.ts"; import type { Content, Kind, LedgerRecord, SubmissionSlot } from "./kinds.ts"; -// 写入时由 ledger 拒绝的四种情况（docs/结构.md 第四部分"写入时由 ledger 拒绝的几种情况"），在写入的同一个事务里查。 -// 只有这四种。记录之间别的先后（例如运行结果前要先有登记）和提交种类与阶段对不对得上都不查： +// 写入时由 ledger 拒绝的五种情况（docs/结构.md 第四部分"写入时由 ledger 拒绝的几种情况"），在写入的同一个事务里查。 +// 只有这五种。记录之间别的先后（例如运行结果前要先有登记）和提交种类与阶段对不对得上都不查： // flow 调 ledger 是模块之间的调用，不是系统边界。"只发生一次"由 schema.ts 的唯一约束管。 // 提交内容的字段归 submissions，该不该开下一个阶段归 flow。 @@ -16,14 +16,18 @@ export interface WriteState { runStarted(run: string): boolean; /** 这一轮有没有消融格的运行已经确认启动。 */ ablationStarted(round: number): boolean; + /** 这一轮的诊断运行有没有已经确认启动。 */ + diagnosisStarted(round: number): boolean; } -/** 实现和核验阶段的提交不能替换（方法.md 第五部分总原则）：实现、修复、核验、消融实现、消融核验。 */ +/** 实现和核验阶段的提交不能替换（方法.md 第五部分总原则）：实现、修复、核验、消融实现、消融核验、诊断实现（含修复）、诊断核验。 */ const NOT_REPLACEABLE: ReadonlySet = new Set([ "implementation", "verification", "ablationImplementation", "ablationVerification", + "diagnosisImplementation", + "diagnosisVerification", ]); export function checkWrite(kind: Kind, content: unknown, state: WriteState): void { @@ -35,7 +39,7 @@ export function checkWrite(kind: Kind, content: unknown, state: WriteState): voi /** * 第 1 条：只收开着的阶段的提交，即有开阶段记录、没有关阶段记录。 * 第 2 条：新的一版必须正好接在本人在这个阶段、对同一个对象的最新一版后面（第一版是 1）； - * 实现、修复、核验、消融实现、消融核验只能交第一版。 + * 实现、修复、核验、消融实现、消融核验、诊断实现、诊断修复、诊断核验只能交第一版。 */ function checkSubmission(s: Content<"submission">, state: WriteState): void { const where = `提交 ${s.kind}（阶段 ${s.stageId}，作者 ${s.author}，对象 ${s.target ?? "无"}，编号 ${s.item ?? "无"}，第 ${s.version} 版）`; @@ -63,7 +67,10 @@ function checkRunResult(r: Content<"runResult">, state: WriteState): void { } } -/** 第 4 条：这一轮已经有消融格确认启动了，就不能再关解释阶段（方法.md 4.3 ⑭：解释冻结必须早于任何一格启动）。 */ +/** + * 第 4 条：这一轮已经有消融格确认启动了，就不能再关解释阶段（方法.md 4.3 ⑭：解释冻结必须早于任何一格启动）。 + * 第 5 条：这一轮的诊断运行已经确认启动了，就不能再关诊断预测阶段（方法.md 4.3 诊 2：冻结必须早于诊断运行启动，范围才算事前预测）。 + */ function checkStageClose(c: Content<"stageClose">, state: WriteState): void { const open = state.stageOpen(c.stageId); if (open?.content.stage === "explanation" && state.ablationStarted(open.content.round)) { @@ -71,4 +78,9 @@ function checkStageClose(c: Content<"stageClose">, state: WriteState): void { `不能关解释阶段 ${c.stageId}：第 ${open.content.round} 轮已经有消融格确认启动，解释冻结必须早于任何一格启动（方法.md 4.3 ⑭）`, ); } + if (open?.content.stage === "diagnosisPrediction" && state.diagnosisStarted(open.content.round)) { + throw new Error( + `不能关诊断预测阶段 ${c.stageId}：第 ${open.content.round} 轮的诊断运行已经确认启动，诊断预测冻结必须早于诊断运行启动（方法.md 4.3 诊 2）`, + ); + } } diff --git a/src/ledger/schema.ts b/src/ledger/schema.ts index 7554687..8a58f87 100644 --- a/src/ledger/schema.ts +++ b/src/ledger/schema.ts @@ -3,7 +3,7 @@ import type { DatabaseSync } from "node:sqlite"; // 账本的表结构（docs/结构.md 第四部分"全部只追加"）：只有一张表，每一行是序号、种类、内容、内容的哈希。 // 表结构一改就把版本号加 1；旧文件版本号对不上时打开报错，不迁移，也不逐字核对建表语句。 -export const SCHEMA_VERSION = 4; +export const SCHEMA_VERSION = 5; /** 新文件要执行的建表语句，由 Ledger 在一个事务里执行，最后写入版本号。 */ export const CREATE_SCHEMA = [ @@ -33,6 +33,8 @@ export const CREATE_SCHEMA = [ `CREATE UNIQUE INDEX call_ended_once ON records (json_extract(content, '$.call')) WHERE kind = 'callEnded'`, // 结算以轮次为编号：每轮最多采纳一个改法，所以最多一次消融。 `CREATE UNIQUE INDEX settlement_once ON records (json_extract(content, '$.round')) WHERE kind = 'settlement'`, + // 诊断结算同样以轮次为编号：每轮最多一次诊断。 + `CREATE UNIQUE INDEX diagnosis_settlement_once ON records (json_extract(content, '$.round')) WHERE kind = 'diagnosisSettlement'`, `PRAGMA user_version=${SCHEMA_VERSION}`, ]; diff --git a/src/materials/assigned.ts b/src/materials/assigned.ts index 0502432..a99cd11 100644 --- a/src/materials/assigned.ts +++ b/src/materials/assigned.ts @@ -3,7 +3,14 @@ import { type Draft, factorLines, KIND_TEXT, offText, registeredMetrics, taskPoi import type { Board } from "./read.ts"; import { fenced, json } from "./format.ts"; import { unassignedChanges } from "./locations.ts"; -import { clauseKey, factorKey, splitCell, type FrozenProtocol, type LocationTable } from "./shapes.ts"; +import { + clauseKey, + diagnosisId, + factorKey, + splitCell, + type FrozenProtocol, + type LocationTable, +} from "./shapes.ts"; // 实验员、核验员的 5 种任务（方法.md 4.3 ⑨ ⑯）：只看分给他的那个候选或那一格——冻结协议、起点代码、 // 要实现或核验的东西。不给信念本（用户 2026-10-03 决定 3）、不给任何运行成绩、不给别的候选。 @@ -36,8 +43,7 @@ export function protocolMetrics(board: Board, p: FrozenProtocol): HypothesisMetr })]; } -function metricSection(d: Draft, p: FrozenProtocol): void { - const all = protocolMetrics(d.board, p); +function metricSection(d: Draft, all: HypothesisMetric[]): void { d.section("要实现的指标定义", [ all.length === 0 ? "（这份协议没有要实现的假设专用指标）" : json(all), "", @@ -58,15 +64,22 @@ function codeLines(d: Draft): string[] { return [...d.readableLines(), "读这些目录用 read_source（核验员）或 shell（实验员）。"]; } +/** 本轮做过诊断时，诊断程序的代码也能读（只读），冻结协议可以写"照诊断实验 dN 的做法"（方法.md 4.3 ②）；只给代码，不给成绩和实验原文。 */ +function diagnosisReference(d: Draft): string[] { + if (d.board.diagnosisSettlement(d.round) === undefined) return []; + d.read(diagnosisId(d.round), "本轮诊断程序（只读，冻结协议引用诊断实验时作参考）"); + return ["上面还列了本轮诊断程序的代码：冻结协议引用诊断实验时作参考，别把它当成这个候选的起点。"]; +} + /** ⑨ 实现：冻结协议、指标定义、要交的位置表、起点代码（本批基线）。 */ export function implementation(d: Draft): ChecklistOf<"implementation"> { const candidate = d.req.object!; const { p, items } = frozen(d, candidate); taskSection(d, "实验员", `候选 ${candidate}`); d.read(p.baseline, "起点代码（本批基线）"); - d.section("基线代码", [...codeLines(d), "你的工作目录是起点代码的副本，改动都落在工作目录里。"]); + d.section("基线代码", [...codeLines(d), "你的工作目录是起点代码的副本，改动都落在工作目录里。", ...diagnosisReference(d)]); d.section("本任务要处理的东西", [`### 候选 ${candidate} 的冻结协议`, "", json(p)]); - metricSection(d, p); + metricSection(d, protocolMetrics(d.board, p)); locationTableSection(d, p); return { kind: "implementation", ...items }; } @@ -78,7 +91,7 @@ export function repair(d: Draft): ChecklistOf<"repair"> { taskSection(d, "实验员", `候选 ${candidate}`); d.read(p.baseline, "起点代码（本批基线）"); d.read(candidate, "你上一次提交的代码"); - d.section("基线代码", [...codeLines(d), "你的工作目录是上一次提交的代码的副本。"]); + d.section("基线代码", [...codeLines(d), "你的工作目录是上一次提交的代码的副本。", ...diagnosisReference(d)]); const v = d.board.published("verification").filter((s) => s.target === candidate).at(-1)!.content; d.section("本任务要处理的东西", [ "### 退回意见", @@ -102,7 +115,7 @@ export function repair(d: Draft): ChecklistOf<"repair"> { "", json(p), ]); - metricSection(d, p); + metricSection(d, protocolMetrics(d.board, p)); locationTableSection(d, p); return { kind: "repair", ...items }; } @@ -148,7 +161,7 @@ export function verification(d: Draft): ChecklistOf<"verification"> { "", fenced(d.req.changes!.diff, "diff"), ]); - metricSection(d, p); + metricSection(d, protocolMetrics(d.board, p)); return { kind: "verification", unassigned: found.unassigned.map((u) => u.location), ...items }; } @@ -212,3 +225,60 @@ export function ablationVerification(d: Draft): ChecklistOf<"ablationVerificatio ]); return { kind: "ablationVerification", cell: a.cell, factors: a.factors }; } + +// --------------------------------------------------------------------------- +// 诊断的三种任务（方法.md 4.3 诊 4）：对象是本轮诊断，一位实验员把选中的全部实验写进一份诊断程序，一位核验员逐个核对。 +// 只看选中的实验原文和当前基线代码，不给信念本、不给任何成绩。 +// --------------------------------------------------------------------------- + +/** 三种诊断任务共用：任务与本轮、起点代码（当前基线）、选中的实验原文和指标定义；program 是另外要能读的诊断程序的说明。交回选中的实验编号。 */ +function diagnosisTask(d: Draft, role: string, program: string | null): string[] { + d.group("分给你的材料", "下面是分给你的本轮诊断的全部材料。"); + const chosen = d.board.decision("selectDiagnosis", d.round)!.output.chosen; + const experiments = d.board.diagnosisExperiments(d.round).filter((e) => chosen.includes(e.id)); + taskSection(d, role, `本轮诊断（对象 ${diagnosisId(d.round)}）`); + d.read(d.board.baselineOf(d.round), "起点代码（当前基线）"); + if (program !== null) d.read(diagnosisId(d.round), program); + d.section("基线代码", codeLines(d)); + d.section("选中的诊断实验", [ + "每个实验都从当前基线的代码出发，只动它写的那一处，其余照基线；每个实验交出自己的答案。", + "", + ...experiments.flatMap((e) => [`### ${e.id} ${e.name}`, "", `动哪一处：${e.spot}`, "", e.body, ""]), + ]); + metricSection(d, experiments.flatMap((e) => e.metrics)); + return chosen; +} + +/** 诊断实现，加上原始产物的位置；修复（repair 为 true）另放退回意见（没通过的实验和依据、上一次的位置表和改动），通过的实验不要动。 */ +function diagnosisCode(d: Draft, repair: boolean): string[] { + const chosen = diagnosisTask(d, "实验员", repair ? "你上一次提交的诊断程序" : null); + d.section("原始产物", [ + `产出当前基线的那次正式运行（${d.board.runOf(d.board.baselineOf(d.round))!.plan.run}）的原始产物已经只读地放进你的工作目录，实验要接着算的答案就在里面；位置和怎么读写在任务说明里（research_query 查 task）。`, + "你的工作目录是当前基线代码的副本，加上这份原始产物，改动都落在工作目录里。", + ]); + if (!repair) return chosen; + const v = d.board.published("diagnosisVerification").filter((s) => s.round === d.round).at(-1)!.content; + const failed = v.experiments.filter((x) => !x.followed || !x.nothingElse); + d.section("退回意见", [ + ...failed.map((x) => `- 实验 ${x.experiment}：${x.followed ? "" : "没照写法做；"}${x.nothingElse ? "" : "动了写法以外的地方；"}${x.where}`), + "", + `核验员的说明：${v.body}`, + "只修上面没通过的实验，通过的实验的代码不要动。", + "", + ...["### 上一次交的位置表", "", json(d.board.code(diagnosisId(d.round)).locations), "", "### 上一次提交相对起点代码的改动", "", fenced(d.req.changes!.diff, "diff")], + ]); + return chosen; +} + +export const diagnosisImplementation = (d: Draft): ChecklistOf<"diagnosisImplementation"> => ({ kind: "diagnosisImplementation", experiments: diagnosisCode(d, false) }); +export const diagnosisRepair = (d: Draft): ChecklistOf<"diagnosisRepair"> => ({ kind: "diagnosisRepair", experiments: diagnosisCode(d, true) }); + +/** 诊断核验：选中的实验原文、当前基线到诊断程序的改动、位置表。Jev 的判断不给核验员（同正向核验）。 */ +export function diagnosisVerification(d: Draft): ChecklistOf<"diagnosisVerification"> { + const chosen = diagnosisTask(d, "核验员", "要核验的诊断程序"); + d.section("本任务要处理的东西", [ + ...["### 实验员交的位置表", "", json(d.board.code(diagnosisId(d.round)).locations), ""], + ...["### 改动全文（诊断程序相对当前基线，每段前后各 3 行上下文）", "", fenced(d.req.changes!.diff, "diff")], + ]); + return { kind: "diagnosisVerification", experiments: chosen }; +} diff --git a/src/materials/belief-book.ts b/src/materials/belief-book.ts index 60abb51..6d93dce 100644 --- a/src/materials/belief-book.ts +++ b/src/materials/belief-book.ts @@ -12,6 +12,7 @@ import { type Phase, } from "../rules/index.ts"; import type { ProcessResult, Range } from "../vocab/index.ts"; +import { diagnosisHistory } from "./diagnosis.ts"; import { measured, num, pct, PLACEMENT_TEXT, rangeText, resourcesText, score, SYNERGY_TEXT, UNTESTED_TEXT } from "./format.ts"; import type { StudyFacts } from "./inputs.ts"; import { answerPathLines } from "./locations.ts"; @@ -21,6 +22,8 @@ import { CELL_NOT_RUN, cellId, factorId, factorKey, ORIGINAL_BASELINE, type Sett // 信念本（方法.md 6.10）：程序从已公开的记录现算，不存。数字全由 rules 的函数算，这里只挑出来写成字。 // 只读这几样：已关阶段的提交、按 Board.runPublic 公开的运行、决定、结算、解封；封存、第一次预测、退回一律不读。 +const PHASE_TEXT: Readonly> = { forward: "正向", diagnosis: "诊断", ablation: "消融" }; + /** 范围怎么换算成把握（方法.md 6.2、6.3），写在开头。只说程序怎么用这三个数，不要求研究者照着算。 */ function rangeToBelief(lambda: number): string { return ( @@ -118,6 +121,9 @@ export function beliefBookOf(board: Board, study: StudyFacts, round: number | nu } for (const s of settlements) lines.push(...ablationSummary(board, s)); + // 3 诊断历史 + lines.push("", "**诊断历史**（每轮的诊断实验与实测、每条解释确认或否定、没做的实验和原因，以及程序推出的方向）", ...diagnosisHistory(board, study, latestRound)); + // 4 战绩 lines.push("", "**战绩**（只显示，不改任何权重；偏高偏低都按分数原值判）"); const samples: Array<{ researcher: string; phase: Phase; range: Range; value: number }> = batches.flatMap((d) => @@ -127,6 +133,9 @@ export function beliefBookOf(board: Board, study: StudyFacts, round: number | nu return v === null ? [] : c.ranges.map((r) => ({ researcher: r.researcher, phase: "forward" as const, range: r.range, value: v })); }), ); + for (const s of board.diagnosisSettlements()) + for (const e of s.input.experiments) + if (e.outcome.kind === "valid") for (const x of s.input.explanations) samples.push({ researcher: x.author, phase: "diagnosis", range: x.experiments.find((y) => y.experiment === e.experiment)!.range, value: e.outcome.score }); for (const s of settlements) for (const c of s.input.cells) { if (c.outcome.kind !== "valid") continue; @@ -139,7 +148,7 @@ export function beliefBookOf(board: Board, study: StudyFacts, round: number | nu if (tallies.length === 0) lines.push("- 还没有实测可以核对"); for (const { researcher, phase, tally: t } of tallies) { lines.push( - `- ${researcher}，${phase === "forward" ? "正向" : "消融"} ${t.total} 次：中位数偏高 ${t.medianHigh}、偏低 ${t.medianLow}；` + + `- ${researcher}，${PHASE_TEXT[phase]} ${t.total} 次：中位数偏高 ${t.medianHigh}、偏低 ${t.medianLow}；` + `穿过上沿 ${t.aboveTop}、下沿 ${t.belowBottom}；包住 ${t.covered}；落点 ` + (Object.keys(PLACEMENT_TEXT) as Array).map((k) => `${PLACEMENT_TEXT[k]} ${t.placements[k]}`).join("，"), ); diff --git a/src/materials/blocks.ts b/src/materials/blocks.ts index 45dc71a..b9734e9 100644 --- a/src/materials/blocks.ts +++ b/src/materials/blocks.ts @@ -2,11 +2,12 @@ import type { Ledger } from "../ledger/index.ts"; import { adoptionLine, compareIds, runBalance } from "../rules/index.ts"; import type { Checklist, SubmissionKind } from "../vocab/index.ts"; import { beliefBookOf } from "./belief-book.ts"; -import { directionText, json, measured, score } from "./format.ts"; +import { diagnosisLines } from "./diagnosis.ts"; +import { directionText, json, measured, num, resourcesText, score } from "./format.ts"; import type { TaskRequest } from "./inputs.ts"; import { answerPathLines } from "./locations.ts"; import { Board, type Run } from "./read.ts"; -import { entryId, explanationId, factorKey, splitCell, type FrozenProtocol } from "./shapes.ts"; +import { entryId, explanationId, factorKey, ORIGINAL_BASELINE, splitCell, type FrozenProtocol } from "./shapes.ts"; // 任务材料的形状，和研究者各任务共用的几块（方法.md 第三部分"给研究者的材料顺序"）。 @@ -38,6 +39,11 @@ export const KIND_TEXT: Readonly> = { entry: "第零轮闭门写入口", discussion: "第零轮讨论", claim: "第零轮认领入口", + diagnosisProposal: "写诊断解释与诊断实验", + diagnosisPrediction: "诊断预测", + diagnosisImplementation: "实现诊断实验", + diagnosisRepair: "按核验意见修复诊断程序", + diagnosisVerification: "核验诊断程序", proposal: "提案", firstPrediction: "第一次预测", critique: "挑刺", @@ -142,12 +148,42 @@ export function codeSection(d: Draft): void { d.read(d.board.baselineOf(d.round), "本批基线"); for (const r of d.board.runs()) { if (r.plan.object !== null && r.plan.purpose !== "baseline" && d.board.runPublic(r)) { - d.read(r.plan.object, `${r.plan.purpose === "forward" ? "候选" : "消融格"} ${r.plan.object} 的代码`); + d.read(r.plan.object, r.plan.purpose === "diagnosis" ? `第 ${r.plan.round} 轮诊断程序的代码` : `${r.plan.purpose === "forward" ? "候选" : "消融格"} ${r.plan.object} 的代码`); } } d.section("基线", [...baselineLines(d), "", "能读的代码（用 read_source 读）：", ...d.readableLines()]); } +/** 本轮诊断结果（方法.md 第三部分材料顺序第 3 条）：从提案起的全部研究者任务都有，没诊断写原因；诊断任务自己读时这一段还没走完，不放。 */ +export function diagnosisSection(d: Draft): void { + if (d.board.diagnosisDone(d.round)) d.section("本轮诊断结果", diagnosisLines(d.board, d.req.study, d.round, true)); +} + +/** 运行的产物只放进查询，这里列编号。 */ +export function artifactSection(d: Draft, heading: string): void { + const artifacts = d.req.artifacts!; + const listed = Object.keys(artifacts) + .sort(compareIds) + .flatMap((run) => Object.entries(artifacts[run]!).map(([file, text]) => `- ${d.record(`${run}/${file}`, `运行 ${run} 的产物 ${file}`, text)}`)); + d.section(heading, ["用 research_query 按编号查全文：", ...(listed.length === 0 ? ["（没有）"] : listed)]); +} + +/** 本批基线那次运行的结果：分数、判分、其余指标、资源、任务的附加记录；原始基线没有位置表，不写答案路径。 */ +export function baselineRunSection(d: Draft): void { + const base = d.board.baselineOf(d.round); + const run = d.board.publicRunOf(base)!; + const j = run.result!.judged!; + const others = Object.keys(j.metrics).filter((k) => k !== d.req.study.finalMetric).sort(compareIds); + d.section("本批基线那次运行的结果", [ + ...(base === ORIGINAL_BASELINE + ? [`运行 ${run.plan.run}：任务给的原始代码；${j.verdict === "valid" ? "算数" : "不算数"}（${j.verdictReason}）；分数 ${measured(j.score, score)}`] + : runLines(d, run)), + resourcesText(run.result!.process, d.board.publicRunOf(ORIGINAL_BASELINE)!.result!.process, d.req.study.wallSeconds), + ...(others.length === 0 ? [] : [`其余指标：${others.map((k) => `${k} ${measured(j.metrics[k]!, num)}`).join("；")}`]), + `任务的附加记录（含合法化，原样）：${JSON.stringify(j.extra)}`, + ]); +} + export function beliefSection(d: Draft): void { d.section("信念本", [beliefBookOf(d.board, d.req.study, d.round)]); } @@ -194,7 +230,13 @@ export function publicRecords(d: Draft): void { e.content.explanations.forEach((x, i) => d.record(explanationId(e.round, e.author, i + 1), `解释 ${explanationId(e.round, e.author, i + 1)}（${e.author}）`, x)); } for (const w of b.published("wrapUp")) d.record(`${w.stageId}.${w.author}`, `${w.author} 第 ${w.round} 轮的收尾`, w.content.body); - for (const r of b.runs()) if (r.plan.purpose !== "baseline" && b.runPublic(r)) d.record(r.plan.run, `运行 ${r.plan.run}`, [...runLines(d, r), "", json(r.result!.judged)].join("\n")); + for (const r of b.runs()) { + if (r.plan.purpose !== "baseline" && r.plan.purpose !== "diagnosis" && b.runPublic(r)) d.record(r.plan.run, `运行 ${r.plan.run}`, [...runLines(d, r), "", json(r.result!.judged)].join("\n")); + } + // 诊断的完整结果：各人给的三个数（结算输入）不放，只有合并后的范围和逐个实验的判分。 + for (const { round, input: _input, ...settled } of b.diagnosisSettlements()) { + d.record(`diagnosis-r${round}`, `第 ${round} 轮诊断的完整结果`, json({ explanations: b.diagnosisExplanations(round), experiments: b.diagnosisExperiments(round), ...settled })); + } d.section("研究记录", [ "已经公开的记录，一行一条；全文用 research_query 按编号查。", ...d.records.map((r) => `- ${r.id}：${r.title}`), diff --git a/src/materials/build.ts b/src/materials/build.ts index 5dc9dd4..d53a020 100644 --- a/src/materials/build.ts +++ b/src/materials/build.ts @@ -12,6 +12,11 @@ const BUILDERS: { [K in SubmissionKind]: (d: Draft) => Extract c.net)))}`; + return `选中的实验核验都没通过、修复次数用完，一个都没跑：${board.eventReason(DIAGNOSIS_NOT_RUN, diagnosisId(round))}`; +} + +const signed = (x: number): string => (x > 0 ? `+${score(x)}` : score(x)); + +/** 一个合并后的实验：做了什么、实测、结局、测前合并范围、比特、惊讶、用时。 */ +function experimentLine(board: Board, study: StudyFacts, round: number, s: DiagnosisSettlement, e: MergedExperiment): string { + const row = s.output.experiments.find((x) => x.experiment === e.id)!; + const judged = s.judged.experiments.find((x) => x.id === e.id); + const dropped = s.dropped.find((x) => x.experiment === e.id); + const skipped = board.decision("selectDiagnosis", round)!.output.experiments.find((y) => y.experiment === e.id)!; + const outcome = { + valid: () => `实测 ${measured(judged!.score, score)}，改进量 ${signed(row.gain!)}`, + untested: () => `没做：${SKIPPED[skipped.reason!]}${skipped.bestCombo === null ? "" : `（含它的最好组合净值 ${num(skipped.bestCombo.net)}）`}`, + insufficient: () => `证据不足：${dropped === undefined ? `${ENDING[judged!.ending]}；${judged!.verdictReason}` : `核验不通过，${dropped.reason}`}`, + }[row.outcome](); + const parts = [outcome, `测前合并范围 ${rangeText(row.merged)}`]; + if (row.worldBits !== null) parts.push(`世界比特 ${num(row.worldBits)}`); + if (row.surprise !== null) parts.push(`惊讶 ${measured(row.surprise.bits, (x) => `${num(x)} 比特`)}`); + if (judged !== undefined) { + parts.push(`用时 ${measured(judged.wallSeconds, (x) => `${num(x)} 秒`)}`); + const others = Object.keys(judged.metrics).filter((k) => k !== study.finalMetric).sort(); + if (others.length > 0) parts.push(`其余指标：${others.map((k) => `${k} ${measured(judged.metrics[k]!, num)}`).join("；")}`); + } + return `- ${e.id} ${e.name}（作者 ${e.authors.join("、")}；动 ${e.spot}）：${parts.join("；")}`; +} + +/** + * 第 round 轮诊断的结果：每个实验做了什么、实测、用时、结局，每条诊断解释确认或否定，程序推出的方向；没诊断时写原因。 + * detail 为 true 时每条解释带正文（公共材料），否则只列确认与否定（信念本的诊断历史）。 + */ +export function diagnosisLines(board: Board, study: StudyFacts, round: number, detail: boolean): string[] { + const s = board.diagnosisSettlement(round); + if (s === undefined) return [`本轮没有诊断：${noDiagnosisReason(board, study, round)}`]; + const process = board.runs().find((r) => r.plan.run === s.run)!.result!.process; + const lines = [ + `诊断运行 ${s.run}：在基线 ${board.baselineOf(round)}（最终指标 ${score(s.input.baseline)}）上，一次运行做了 ${s.judged.experiments.length} 个实验，只扣 1 次。` + + "诊断实验的答案不采纳为基线，只用来看这个方向还剩多少空间。", + resourcesText(process, board.publicRunOf(ORIGINAL_BASELINE)!.result!.process, study.wallSeconds), + ...["", "诊断实验：", ...board.diagnosisExperiments(round).map((e) => experimentLine(board, study, round, s, e)), "", "诊断解释："], + ]; + for (const x of board.diagnosisExplanations(round)) { + const verdicts = s.output.judgments.find((j) => j.id === x.id)!.verdicts; + const of = (v: string) => verdicts.filter((y) => y.verdict === v).map((y) => y.experiment).join("、") || "无"; + lines.push(`- ${x.id}（作者 ${x.author}）：${verdicts.length === 0 ? "没有有效实验可判" : `确认 ${of("confirmed")}；否定 ${of("denied")}`}`); + if (detail) lines.push(...x.body.split("\n").map((l) => ` ${l}`)); + } + const d = s.output.direction; + const gain = d.gains.find((g) => g.experiment === d.experiment)!; + const way = `程序按诊断推出的方向：${d.experiment}（${gain.measured ? "实测" : "按更新后的权重估计"} ${signed(d.gain)}）。这只是按公式推出来的参考，你可以不跟。`; + return [...lines, "", d.found ? way : "诊断没有指出能涨分的方向：所有实验的收益都不为正。"]; +} + +/** 信念本的"诊断历史"（方法.md 6.10 第 3 条）：每个诊断这一段已走完的轮次一段。 */ +export function diagnosisHistory(board: Board, study: StudyFacts, latestRound: number): string[] { + const rounds = Array.from({ length: latestRound }, (_, i) => i + 1).filter((r) => board.diagnosisDone(r)); + return rounds.length === 0 ? ["- 还没有"] : rounds.flatMap((r) => [`- 第 ${r} 轮：`, ...diagnosisLines(board, study, r, false).map((l) => (l === "" ? l : ` ${l}`))]); +} diff --git a/src/materials/index.ts b/src/materials/index.ts index 2764725..c0da607 100644 --- a/src/materials/index.ts +++ b/src/materials/index.ts @@ -7,7 +7,7 @@ export { buildMaterial } from "./build.ts"; export type { CodeChanges, PromptFile, Prompts, StudyFacts, TaskRequest } from "./inputs.ts"; export { queryMaterial, readPrompts } from "./prompts.ts"; // flow 推进时读账本也用这一份（本批基线、基线分数、留池候选、各候选合并了哪些原提案），不另写一份。 -export { Board, validScore } from "./read.ts"; +export { Board, validScore, type MergedExperiment } from "./read.ts"; export { callId, candidateId, @@ -17,6 +17,7 @@ export { clauseKey, critiqueId, DECISION_KINDS, + DIAGNOSIS_NOT_RUN, diagnosisExplanationId, diagnosisId, diagnosisKey, diagnosisNotRunEvent, entryId, experimenterId, verifierId, @@ -33,6 +34,7 @@ export { type DecisionKind, type DecisionShapes, type DedupOutput, + type DiagnosisDedupOutput, type DiagnosisLocationTable, type DiagnosisSettlement, type FrozenProtocol, type LocationTable, type Settlement, diff --git a/src/materials/inputs.ts b/src/materials/inputs.ts index a526b7e..01c6d46 100644 --- a/src/materials/inputs.ts +++ b/src/materials/inputs.ts @@ -17,8 +17,10 @@ export type StudyFacts = { lambda: number; /** 正式运行总次数。 */ totalRuns: number; - /** 每人每轮提案上限 M。 */ + /** 每人每轮提案上限 M，诊断解释的上限也是它。 */ proposalLimit: number; + /** 每人每轮诊断实验上限 A；诊断关掉时为 null。 */ + diagnosisExperimentLimit: number | null; factorLimit: number; /** 最终指标和常规辅助指标的定义（tasks 的 metricDefinitions）。 */ metrics: MetricDefinition[]; @@ -50,15 +52,15 @@ export type Prompts = { roles: Record; tasks: Record/<版本号>`，flow 照材料里的目录取出。 */ export type TaskRequest = { kind: SubmissionKind; actor: string; stageId: string; - /** 一个任务一个对象：候选、原提案（挑刺）或消融格；入口、讨论、志愿、提案、解释、收尾为 null。 */ + /** 一个任务一个对象：候选、原提案（挑刺）、消融格或本轮诊断（diagnosisId）；入口、讨论、志愿、提案、诊断解释与预测、解释、收尾为 null。 */ object: string | null; ledger: Ledger; study: StudyFacts; diff --git a/src/materials/read.ts b/src/materials/read.ts index a4ef581..74b70be 100644 --- a/src/materials/read.ts +++ b/src/materials/read.ts @@ -1,11 +1,13 @@ import type { Content, LedgerRecord, StoredSubmission } from "../ledger/index.ts"; import { compareIds, type RunState } from "../rules/index.ts"; -import type { JudgedResult, Range, Stage, SubmissionKind } from "../vocab/index.ts"; +import type { HypothesisMetric, JudgedResult, Range, Stage, SubmissionKind } from "../vocab/index.ts"; import { cellId, + diagnosisExplanationId, ORIGINAL_BASELINE, type DecisionKind, type DecisionShapes, + type DiagnosisSettlement, type FrozenProtocol, type LocationTable, type Settlement, @@ -22,6 +24,9 @@ export type Run = { result: Content<"runResult"> | null; }; +/** 本轮合并后的一个诊断实验：编号、全部作者、正文（取作者里编号最小的那一份）、登记的指标（编号已是全名"实验编号.短名"）。 */ +export type MergedExperiment = { id: string; authors: string[]; name: string; spot: string; body: string; metrics: HypothesisMetric[] }; + type StageInfo = { stage: Stage; round: number; roster: string[]; closed: boolean }; export class Board { @@ -116,12 +121,14 @@ export class Board { /** * 运行什么时候公开（任务书 09 设计稿）：基线运行有结果即公开；第 r 轮正向运行在第 r 轮的采纳决定写入后 - * 整批公开（方法.md 4.3 ⑪）；第 r 轮消融运行在第 r 轮结算写入后公开（4.3 ⑰ ⑱）。 + * 整批公开（方法.md 4.3 ⑪）；第 r 轮消融运行在第 r 轮结算写入后公开（4.3 ⑰ ⑱）； + * 第 r 轮诊断运行在第 r 轮诊断结算写入后公开（4.3 诊 4）。 */ runPublic(run: Run): boolean { if (run.result === null) return false; if (run.plan.purpose === "baseline") return true; if (run.plan.purpose === "forward") return this.decision("adopt", run.plan.round) !== undefined; + if (run.plan.purpose === "diagnosis") return this.diagnosisSettlement(run.plan.round) !== undefined; return this.settlement(run.plan.round) !== undefined; } @@ -140,7 +147,12 @@ export class Board { /** 每次运行在账上的状态，算余额用（方法.md 第七部分）。 */ runStates(): RunState[] { - return this.runs().map((r) => (r.result?.process.ending === "notStarted" ? "notStarted" : r.started ? "started" : "reserved")); + return this.runs().map(stateOf); + } + + /** 第 round 轮开始前各次运行在账上的状态（算诊断开始时的余额）。 */ + runStatesBefore(round: number): RunState[] { + return this.runs().filter((r) => r.plan.round < round).map(stateOf); } /** 某个候选最新的实现或修复提交（已关阶段）。 */ @@ -169,6 +181,38 @@ export class Board { return this.#of("settlement").map((r) => ({ round: r.content.round, ...(r.content.result as unknown as Settlement) })); } + /** 第 round 轮的诊断结算（还没写入为 undefined）。 */ + diagnosisSettlement(round: number): DiagnosisSettlement | undefined { + return this.diagnosisSettlements().find((s) => s.round === round); + } + + diagnosisSettlements(): Array<{ round: number } & DiagnosisSettlement> { + return this.#of("diagnosisSettlement").map((r) => ({ round: r.content.round, ...(r.content.result as unknown as DiagnosisSettlement) })); + } + + /** 第 round 轮诊断这一段走完了没有：提案阶段开了，就是结算写了或本轮不诊断。 */ + diagnosisDone(round: number): boolean { + return [...this.stages.values()].some((s) => s.stage === "proposal" && s.round === round); + } + + /** 第 round 轮已公开的诊断解释，按作者、条序排。 */ + diagnosisExplanations(round: number): Array<{ id: string; author: string; body: string }> { + return this.published("diagnosisProposal") + .filter((s) => s.round === round) + .sort((a, b) => compareIds(a.author, b.author)) + .flatMap((s) => s.content.explanations.map((x, i) => ({ id: diagnosisExplanationId(round, s.author, i + 1), author: s.author, body: x.body }))); + } + + /** 第 round 轮合并后的诊断实验：判重合并的决定指明合成自哪几个作者的哪几个实验，正文取第一个。 */ + diagnosisExperiments(round: number): MergedExperiment[] { + const proposals = this.published("diagnosisProposal").filter((s) => s.round === round); + const experimentOf = (src: { author: string; index: number }) => proposals.find((s) => s.author === src.author)!.content.experiments[src.index]!; + return this.decision("diagnosisDedup", round)!.output.experiments.map(({ experiment, sources }) => { + const { name, spot, body, metrics } = experimentOf(sources[0]!); + return { id: experiment, authors: [...new Set(sources.map((x) => x.author))], name, spot, body, metrics: metrics.map((m) => ({ ...m, id: `${experiment}.${m.id}` })) }; + }); + } + /** 本人最新的私人笔记（方法.md 第三部分"私人笔记"），没写过为 null。 */ note(author: string): string | null { return this.#of("note").filter((r) => r.content.author === author).at(-1)?.content.body ?? null; @@ -249,6 +293,8 @@ export class Board { } } +const stateOf = (r: Run): RunState => (r.result?.process.ending === "notStarted" ? "notStarted" : r.started ? "started" : "reserved"); + /** 判分结果里的有效分数：算数且测到了分数才有，否则 null。 */ export function validScore(judged: JudgedResult | null): number | null { return judged !== null && judged.verdict === "valid" && judged.score.status === "measured" ? judged.score.value : null; diff --git a/src/materials/researcher.ts b/src/materials/researcher.ts index 55727f4..e1d49dc 100644 --- a/src/materials/researcher.ts +++ b/src/materials/researcher.ts @@ -1,8 +1,11 @@ import { compareIds } from "../rules/index.ts"; import { ANSWER_PATH_KEY, SAFEGUARD_TRIGGERS_KEY, type ChecklistOf } from "../vocab/index.ts"; import { + artifactSection, + baselineRunSection, beliefSection, codeSection, + diagnosisSection, Draft, entryVersions, factorLines, @@ -33,6 +36,7 @@ function frame(d: Draft, own: Own, middle: () => void): void { researcherTaskSection(d); codeSection(d); if (d.round >= 1) { + diagnosisSection(d); publicRecords(d); beliefSection(d); } @@ -126,16 +130,42 @@ function concernLines(d: Draft, round: number): string[] { return lines.length === 0 ? ["（没有）"] : lines; } +/** 上一轮全部研究者的收尾（提案、诊 1 共用）。 */ +function previousWrapSection(d: Draft): void { + const prev = d.round - 1; + const wraps = d.board.published("wrapUp").filter((w) => w.round === prev); + d.section("上一轮的收尾", prev < 1 ? ["（这是第 1 轮，没有上一轮）"] : wraps.flatMap((w) => [`### ${w.author} 的收尾`, "", w.content.body, ""])); +} + +/** 诊 1 诊断解释与诊断实验：当前基线的代码，产出它的那次运行的答案、分数、指标，信念本，上一轮收尾；第 1 轮的第零轮全部记录在研究记录里。 */ +export function diagnosisProposal(d: Draft): ChecklistOf<"diagnosisProposal"> { + frame(d, [], () => { + baselineRunSection(d); + artifactSection(d, "本批基线那次运行的答案"); + previousWrapSection(d); + }); + return { kind: "diagnosisProposal", maxExplanations: d.req.study.proposalLimit, maxExperiments: d.req.study.diagnosisExperimentLimit!, takenMetricIds: factorRules(d).takenMetricIds }; +} + +/** 诊 2 诊断预测：合并后的诊断实验清单（标明作者）、全部诊断解释；自己的解释另列在"你自己的"里。别人在这一步交的数一概不读（只经 Board.mine 读本人的）。 */ +export function diagnosisPrediction(d: Draft): ChecklistOf<"diagnosisPrediction"> { + const all = d.board.diagnosisExplanations(d.round); + const mine = all.filter((x) => x.author === d.req.actor); + const experiments = d.board.diagnosisExperiments(d.round); + const text = (x: { id: string; author: string; body: string }) => [`#### ${x.id}（作者 ${x.author}）`, "", x.body, ""]; + frame(d, [{ heading: "你的诊断解释", lines: ["下面每一条都要对每个合并后的诊断实验各给三个数。", "", ...mine.flatMap(text)] }], () => { + baselineRunSection(d); + const merged = experiments.flatMap((e) => [`#### ${e.id} ${e.name}（作者 ${e.authors.join("、")}）`, "", `动哪一处：${e.spot}`, "", e.body, ""]); + d.section("本任务要处理的东西", ["### 合并后的诊断实验", "", ...merged, "### 全部诊断解释（含你自己的）", "", ...all.flatMap(text)]); + }); + return { kind: "diagnosisPrediction", explanations: mine.map((x) => x.id), experiments: experiments.map((e) => e.id) }; +} + /** ② 提案：公共材料，加上一轮全部收尾、上一轮实验员的提醒、可以写进"延续"的因子。 */ export function proposal(d: Draft): ChecklistOf<"proposal"> { frame(d, [], () => { - const prev = d.round - 1; - const wraps = d.board.published("wrapUp").filter((w) => w.round === prev); - d.section( - "上一轮的收尾", - prev < 1 ? ["（这是第 1 轮，没有上一轮）"] : wraps.flatMap((w) => [`### ${w.author} 的收尾`, "", w.content.body, ""]), - ); - d.section("上一轮实验员实现时的提醒", prev < 1 ? ["（这是第 1 轮，没有上一轮）"] : concernLines(d, prev)); + previousWrapSection(d); + d.section("上一轮实验员实现时的提醒", d.round < 2 ? ["（这是第 1 轮，没有上一轮）"] : concernLines(d, d.round - 1)); const lines = d.board.protocols().flatMap((p) => p.protocol.factors.map((f, i) => `- ${factorId(p.candidate, factorKey(i))} ${f.name}（关键设定：${f.keySettings}；候选 ${p.candidate}，基线 ${p.protocol.baseline}${f.continues === null ? "" : `；它自己延续 ${f.continues}`}）`), ); @@ -258,11 +288,7 @@ export function explanation(d: Draft): ChecklistOf<"explanation"> { "", ...factorLines(p), ]); - const artifacts = d.req.artifacts!; - const listed = Object.keys(artifacts) - .sort(compareIds) - .flatMap((run) => Object.entries(artifacts[run]!).map(([file, text]) => `- ${d.record(`${run}/${file}`, `运行 ${run} 的产物 ${file}`, text)}`)); - d.section("本轮运行的产物", ["用 research_query 按编号查全文：", ...(listed.length === 0 ? ["（没有）"] : listed)]); + artifactSection(d, "本轮运行的产物"); }); return { kind: "explanation", cells: p.factors.map((_, i) => factorKey(i)), maxExplanations: d.req.study.proposalLimit }; } diff --git a/src/materials/shapes.ts b/src/materials/shapes.ts index 373be7e..77ff694 100644 --- a/src/materials/shapes.ts +++ b/src/materials/shapes.ts @@ -5,13 +5,17 @@ import type { BetResult, CellDecision, CellRuleInput, + DiagnosisDecision, + DiagnosisRuleInput, EntryAssignment, + SettleDiagnosisInput, SettleInput, adopt, settleAblation, + settleDiagnosis, } from "../rules/index.ts"; import type { Content } from "../ledger/index.ts"; -import type { Json, Role, Submission, SubmissionKind } from "../vocab/index.ts"; +import type { Json, Measured, Role, Submission, SubmissionKind, Verdict } from "../vocab/index.ts"; // 账本里只存成 Json 的几样内容长什么样、编号怎么起（用户 2026-10-03 决定 2）：materials 读它们，flow 照着写。 // 决定和结算的输出直接用 rules 的返回类型；ledger 不认识这些类型，只存 Json。 @@ -51,6 +55,12 @@ export function splitCell(cell: string): { candidate: string; key: string } { } /** 解释：第 round 轮作者交的第 i 条（从 1 数）。结算里判确认、否定用它。 */ export const explanationId = (round: number, author: string, i: number): string => `r${round}-${author}-x${i}`; +/** 本轮诊断：诊断程序的冻结代码、诊断运行、诊断实现与核验的对象都是它。 */ +export const diagnosisId = (round: number): string => `r${round}-diagnosis`; +/** 诊断实验在本轮合并后清单里的编号：第 i 个（从 0 数）是 d(i+1)；它登记的指标全名是"实验编号.短名"。 */ +export const diagnosisKey = (i: number): string => `d${i + 1}`; +/** 诊断解释：第 round 轮作者交的第 i 条（从 1 数）。 */ +export const diagnosisExplanationId = (round: number, author: string, i: number): string => `r${round}-${author}-dx${i}`; /** 模型调用：同一个任务（阶段、谁、哪种提交、对象）第 attempt 次发起。 */ export const callId = (stageId: string, actor: string, kind: string, object: string | null, attempt: number): string => `${stageId}.${actor}.${kind}.${object ?? "none"}.${attempt}`; @@ -104,11 +114,19 @@ export function freezeProtocol(body: ProtocolBody, candidate: string, round: num /** 候选代码的位置表，就是实验员交的那三项；基线和消融格的代码记 null（消融格用完整代码的位置表）。 */ export type LocationTable = Pick, "factorLocations" | "safeguardLocations" | "unassigned">; +/** 诊断程序的位置表：每个选中的实验一项（诊断实现交的 locations）。 */ +export type DiagnosisLocationTable = Submission<"diagnosisImplementation">["locations"]; // --------------------------------------------------------------------------- // 决定（账本记录 decision：decision 是种类，input、output 如下） // --------------------------------------------------------------------------- +/** + * 诊断实验判重合并的输出（方法.md 4.3 诊 2）：合并后的每个实验由哪几个作者的哪几个实验（index 是它在作者那份诊断提案里的位置，从 0 数）合成； + * sources 按作者编号排，第一个就是"作者里编号最小的那一份"，合并后的正文取它的。 + */ +export type DiagnosisDedupOutput = { experiments: Array<{ experiment: string; sources: Array<{ author: string; index: number }> }> }; + /** 判重合并的输出（方法.md 4.3 ③）：本轮新候选各合并了哪几条提案；留池候选各并进了哪几条新提案（可以没有）。 */ export type DedupOutput = { fresh: Array<{ candidate: string; proposals: string[] }>; @@ -127,9 +145,11 @@ export type DecisionShapes = { adopt: { input: Parameters[0]; output: ReturnType }; cells: { input: CellRuleInput; output: CellDecision }; bet: { input: AblationModelInput & { cells: SettleInput["cells"] }; output: BetResult }; + diagnosisDedup: { output: DiagnosisDedupOutput }; + selectDiagnosis: { input: DiagnosisRuleInput; output: DiagnosisDecision }; }; export type DecisionKind = keyof DecisionShapes; -export const DECISION_KINDS: readonly DecisionKind[] = ["entries", "dedup", "batch", "adopt", "cells", "bet"]; +export const DECISION_KINDS: readonly DecisionKind[] = ["entries", "dedup", "batch", "adopt", "cells", "bet", "diagnosisDedup", "selectDiagnosis"]; // --------------------------------------------------------------------------- // 结算（账本记录 settlement.result，方法.md 4.3 ⑰） @@ -142,6 +162,25 @@ export type Settlement = { output: ReturnType; }; +/** + * 一次诊断的结算（账本记录 diagnosisSettlement.result，方法.md 6.12）。judged 是每个跑过的实验单独判分的结果，与 tasks 的 + * DiagnosisJudged 同形（materials 不引用 tasks）；诊断运行记录的 judged 为 null，逐个实验的结果只在这里。 + * dropped 是核验不通过、修复用完而没跑的实验和原因；input、output 是 settleDiagnosis 的输入输出，没做的实验在里面记 untested。 + */ +export type DiagnosisSettlement = { + run: string; + judged: { + experiments: Array<{ id: string; ending: "finished" | "crashed" | "timeout" | "notRun"; wallSeconds: Measured; verdict: Verdict; verdictReason: string; score: Measured; metrics: Record; extra: Json }>; + }; + dropped: Array<{ experiment: string; reason: string }>; + input: SettleDiagnosisInput; + output: ReturnType; +}; + +/** 事件（账本记录 event）：选中的诊断实验核验都没通过、修复用完，一个都不剩，不跑、不扣次（方法.md 4.3 诊 4）。objects 只有本轮诊断的编号。 */ +export const DIAGNOSIS_NOT_RUN = "diagnosisNotRun"; +export const diagnosisNotRunEvent = (round: number, reason: string): Content<"event"> => ({ event: DIAGNOSIS_NOT_RUN, reason, objects: [diagnosisId(round)] }); + /** * 事件（账本记录 event）：一格消融核验不通过、重做次数用完，这一格不跑、记"证据不足"（方法.md 4.3 ⑯、6.6）。 * 这一格没有运行，信念本从这条事件读原因。objects 只有这一格的编号。 diff --git a/src/rules/cells.ts b/src/rules/cells.ts index 6fc9338..cad634e 100644 --- a/src/rules/cells.ts +++ b/src/rules/cells.ts @@ -1,9 +1,17 @@ import type { AblationCell } from "./ablation.ts"; import { compareIds, improvement, NET_TOLERANCE } from "./basics.ts"; import { ablationCellLimit } from "./budget.ts"; -import { density } from "./distribution.ts"; import { seededSample } from "./random.ts"; -import { ablationModel, evaluateCombos, type AblationModelInput, type ComboParams, type ComboValue } from "./value.ts"; +import { + ablationModel, + ablationView, + evaluateCombos, + itemGain, + updatedWeights, + type AblationModelInput, + type ComboParams, + type ComboValue, +} from "./value.ts"; // 方法.md 6.7"选格"、第十一部分的四条选格规则，以及 6.8 押的因子。 @@ -121,10 +129,15 @@ function netOf(combos: readonly ComboValue[], key: string): number { * "含这一格的净值最大组合"直接按这条挑（只是记录，不要求净值 > 1e-12）。 */ export function topCombo(combos: readonly C[]): C | null { + return topByNet(combos, (c) => c.factors); +} + +/** topCombo 的比较规则，组合里的项（因子或实验）怎么取由 items 给；选格和选诊断实验共用（6.7、6.12）。 */ +export function topByNet(combos: readonly C[], items: (c: C) => readonly string[]): C | null { if (combos.length === 0) return null; const top = Math.max(...combos.map((c) => c.net)); const near = combos.filter((c) => c.net >= top - NET_TOLERANCE); - return near.reduce((best, c) => (compareCombos(c.factors, best.factors) < 0 ? c : best)); + return near.reduce((best, c) => (compareCombos(items(c), items(best)) < 0 ? c : best)); } /** 格数少的在前；格数相同时按因子编号逐个比。 */ @@ -207,19 +220,11 @@ export function chooseBet(input: AblationModelInput & { cells: readonly Ablation const o = input.cells.find((c) => c.factor === f)?.outcome; return o?.kind === "valid" ? o.score : null; }); - const raw = model.weights.map((w, j) => - scores.reduce((v, x, c) => (x === null ? v : v * density(model.rawKnots[c]![j]!, x)), w), - ); - const sum = raw.reduce((t, v) => t + v, 0); - const weightsKept = sum === 0; - const weights = weightsKept ? [...model.weights] : raw.map((v) => v / sum); + const view = ablationView(model); + const { weights, kept: weightsKept } = updatedWeights(view, scores); const gains = model.factors.map((factor, c) => { - const x = scores[c]; - const gain = - x !== null && x !== undefined - ? model.space * (improvement(x, input.full, input.direction) / model.gain) - : model.space * model.expected[c]!.reduce((t, e, j) => t + weights[j]! * e, 0); - return { factor, gain, measured: x !== null }; + const x = scores[c] ?? null; + return { factor, gain: itemGain(view, c, weights, x), measured: x !== null }; }); let best = 0; for (let c = 1; c < gains.length; c++) if (gains[c]!.gain > gains[best]!.gain) best = c; diff --git a/src/rules/diagnosis.ts b/src/rules/diagnosis.ts new file mode 100644 index 0000000..dddfef4 --- /dev/null +++ b/src/rules/diagnosis.ts @@ -0,0 +1,304 @@ +import type { Direction, Measured, Range } from "../vocab/index.ts"; +import { judgeExplanations, type AblationExplanation, type CellOutcome, type ExplanationVerdict } from "./ablation.ts"; +import { combinations, compareIds, improvement, NET_TOLERANCE } from "./basics.ts"; +import { bitWidth, disagreement, surprise, worldBits } from "./bits.ts"; +import { topByNet } from "./cells.ts"; +import { ablationCellMixture, knotsMean, mapKnots, mixtureRange } from "./distribution.ts"; +import { + itemGain, + OPEN_RESERVE, + runCost, + updatedWeights, + usePotential, + valueOf, + type ComboParams, + type ValueModel, +} from "./value.ts"; + +// 方法.md 6.12 诊断：成分模型、方向收益、信息价值、价值减成本、选实验、结算。 +// 成分、权重更新、精确积分、部分格网格、价值减成本的公式都在 value.ts，这里只给出诊断的"实测 → 收益"换算： +// 收益 g(x) = 改进量 x − b（越小越好的任务按 6.1 换成 b − x），另有收益为 0 的"都不走"。 + +/** 一条诊断解释：作者，以及对每个合并后的诊断实验给的三个数（实验做完后的分数，原值）。 */ +export interface DiagnosisExplanation { + id: string; + author: string; + experiments: ReadonlyArray<{ experiment: string; range: Range }>; +} + +export interface DiagnosisModelInput { + /** 本批基线 b。 */ + baseline: number; + direction: Direction; + lambda: number; + /** 合并后的全部诊断实验编号（包括没选进组合的）。 */ + experiments: readonly string[]; + /** 每条解释对每个实验都给了三个数。 */ + explanations: readonly DiagnosisExplanation[]; +} + +export interface DiagnosisModel extends ValueModel { + /** 实验编号，按编号排；ValueModel 里"项的下标"按这个次序。 */ + experiments: string[]; + /** 成分：各条解释的编号（按编号排），最后是开放预留。 */ + components: string[]; +} + +/** + * 成分模型（6.12"成分与权重"）：每条诊断解释一个成分，开放预留一个成分；每个成分对每个实验有一个分布， + * 解释用它自己的三个数补尾巴，开放预留用这个实验自己的均匀分布（6.3"诊断"，和消融一格同一个合并函数）。 + */ +export function diagnosisModel(input: DiagnosisModelInput): DiagnosisModel { + const experiments = [...input.experiments].sort(compareIds); + const explanations = [...input.explanations].sort((a, b) => compareIds(a.id, b.id)); + const mixtures = experiments.map((d) => + ablationCellMixture( + explanations.map((e) => ({ id: e.id, author: e.author, range: e.experiments.find((x) => x.experiment === d)!.range })), + input.lambda, + ), + ); + const scale = (x: number): number => improvement(input.baseline, x, input.direction); + const rawKnots = mixtures.map((m) => m.map((c) => c.knots)); + const scaledKnots = rawKnots.map((row) => row.map((k) => mapKnots(k, scale, input.direction === "minimize"))); + const weights = mixtures[0]!.map((c) => c.weight); + const expected = scaledKnots.map((row) => row.map(knotsMean)); + return { + experiments, + components: [...explanations.map((e) => e.id), OPEN_RESERVE], + weights, + rawKnots, + scaledKnots, + expected, + prior: expected.map((row) => row.reduce((s, e, j) => s + weights[j]! * e, 0)), + space: 1, + floor: 0, + scale, + }; +} + +/** 一个诊断实验组合的价值与成本（6.12"价值减成本"）。 */ +export interface DiagnosisCombo { + /** 组合里的实验，按编号排。 */ + experiments: string[]; + voi: number; + /** R̂ = ⌈(余额 − 1) ÷ 每批上限⌉。 */ + rHat: number; + mPrime: number; + /** 价值 = u × m′ × VOI。 */ + value: number; + /** 成本：诊断只占一次运行，任何组合都一样。 */ + cost: number; + net: number; +} + +export interface DiagnosisParams extends ComboParams { + /** 一次诊断最多做几个实验 D（6.11）。 */ + maxExperiments: number; +} + +/** + * 列出可选组合并算价值、成本、净值（6.12"选实验"第 1、2 条）：余额不到 2 次一个都没有；实验数不超过 D 时只有"全做"， + * 超过 D 时是所有恰好 D 个实验的组合，按实验编号字典序列出。ĝ₀ = max(0, max_d E(g_d))。 + */ +export function evaluateDiagnosisCombos( + model: DiagnosisModel, + params: DiagnosisParams, +): { g0: number; combos: DiagnosisCombo[] } { + const g0 = Math.max(0, ...model.prior); + if (params.balance < 2) return { g0, combos: [] }; + const rHat = Math.ceil((params.balance - 1) / params.batchCap); + const mPrime = usePotential(rHat, params.decay); + const cost = runCost(g0, params.batchCap, 1); + const size = Math.min(model.experiments.length, params.maxExperiments); + const combos = combinations(model.experiments.length, size).map((S) => { + const voi = valueOf(model, S); + const value = params.u * mPrime * voi; + return { experiments: S.map((d) => model.experiments[d]!), voi, rHat, mPrime, value, cost, net: value - cost }; + }); + return { g0, combos }; +} + +export interface DiagnosisRuleInput extends DiagnosisModelInput, DiagnosisParams {} + +/** + * 没选中的实验为什么没做（6.12"选实验"第 4 条）：budget 放不下（已选的组合装满了 D 个）； + * value 信息价值不足以抵过成本。 + */ +export type SkippedReason = "budget" | "value"; + +/** 选诊断实验的结果，含 6.12 要求附上的全部数字；规则名和版本由登记表补上。 */ +export interface DiagnosisDecision { + params: { balance: number; batchCap: number; maxExperiments: number; u: number; decay: number; lambda: number }; + components: string[]; + /** 起始权重，与 components 对齐。 */ + weights: number[]; + /** 各实验的事前期望收益 E(g_d)，按实验编号排。 */ + priorGains: Array<{ experiment: string; gain: number }>; + /** ĝ₀ = max(0, max_d E(g_d))。 */ + g0: number; + /** 可选的全部组合及其价值、成本、净值（余额不到 2 次时为空）。 */ + combos: DiagnosisCombo[]; + /** 本轮做不做诊断：选中了净值 > 1e-12 的组合才做。 */ + diagnose: boolean; + /** 选中要做的实验，按编号排；不诊断时为空。 */ + chosen: string[]; + /** 选中组合的净值；不诊断时是 0。 */ + chosenNet: number; + /** + * 每个合并后的实验：做不做；没做的写原因，并附"含这个实验的组合里净值最大的那个"； + * 没有任何含它的可选组合（余额不到 2 次，或没被任何组合包含）时 bestCombo 为 null。 + */ + experiments: Array<{ + experiment: string; + done: boolean; + reason: SkippedReason | null; + bestCombo: { experiments: string[]; net: number } | null; + }>; +} + +/** + * 选诊断实验，规则"信息价值减成本"（6.12"选实验"）：取净值最大、且净值 > 1e-12 的组合，净值相差不到 1e-12 的 + * 视为相同、按实验编号从小到大比（和选格同一条挑法，见 topByNet）；都不大于 1e-12 时本轮不诊断。 + */ +export function selectDiagnosis(input: DiagnosisRuleInput): DiagnosisDecision { + const model = diagnosisModel(input); + const { g0, combos } = evaluateDiagnosisCombos(model, input); + const best = topByNet(combos.filter((c) => c.net > NET_TOLERANCE), (c) => c.experiments); + const chosen = best === null ? [] : [...best.experiments]; + return { + params: { + balance: input.balance, + batchCap: input.batchCap, + maxExperiments: input.maxExperiments, + u: input.u, + decay: input.decay, + lambda: input.lambda, + }, + components: model.components, + weights: model.weights, + priorGains: model.experiments.map((experiment, d) => ({ experiment, gain: model.prior[d]! })), + g0, + combos, + diagnose: best !== null, + chosen, + chosenNet: best === null ? 0 : best.net, + experiments: model.experiments.map((experiment) => { + const done = chosen.includes(experiment); + const top = topByNet(combos.filter((c) => c.experiments.includes(experiment)), (c) => c.experiments); + return { + experiment, + done, + reason: done ? null : chosen.length > 0 ? "budget" : "value", + bestCombo: done || top === null ? null : { experiments: [...top.experiments], net: top.net }, + }; + }), + }; +} + +/** + * 一个实验的结局（6.12"结算"）：valid 有效分数；insufficient 证据不足（跑崩、超时、没有产物、修不了、核验不通过）； + * untested 没做。和消融一格的三种同形，直接用 ablation.ts 的类型。 + */ +export type ExperimentOutcome = Extract; + +export interface SettleDiagnosisInput { + /** 本批基线 b。 */ + baseline: number; + direction: Direction; + epsilon: number; + lambda: number; + /** 合并后的全部诊断实验及结局，没做的写 untested。 */ + experiments: ReadonlyArray<{ experiment: string; outcome: ExperimentOutcome }>; + explanations: readonly DiagnosisExplanation[]; +} + +/** 诊断推出的方向（6.12"结算"）。 */ +export interface DiagnosisDirection { + /** 更新后的成分权重（与模型的成分对齐）；无从更新时等于起始权重。 */ + weights: number[]; + /** 实测落在所有成分的范围之外、权重之和为 0 时为 true，此时保持起始权重。 */ + weightsKept: boolean; + /** 各实验的收益（按编号排）：measured 为 true 的是有效实验实测换算的，其余是按更新后的权重估的。 */ + gains: Array<{ experiment: string; gain: number; measured: boolean }>; + /** 收益最大的实验（严格更大才替换，平局取编号小的）及其收益。 */ + experiment: string; + gain: number; + /** 最大收益为正才算指出了方向；否则记"诊断没有指出能涨分的方向"。 */ + found: boolean; +} + +/** + * 一次诊断的结算（6.12"结算"、6.9）：每个实验的收益、合并范围，有效实验的世界比特、惊讶、分歧量 + * （格宽 ε/100，从 b 起切）；每条诊断解释在每个有效实验上的确认或否定（和消融同一个判法）；诊断推出的方向。 + * 证据不足和没做的实验不判、不记账。 + */ +export function settleDiagnosis(input: SettleDiagnosisInput): { + experiments: Array<{ + experiment: string; + outcome: ExperimentOutcome["kind"]; + gain: number | null; + merged: Range; + worldBits: number | null; + surprise: { cell: number; prior: number; bits: Measured } | null; + disagreement: number | null; + }>; + judgments: Array<{ id: string; verdicts: Array<{ experiment: string; verdict: ExplanationVerdict }> }>; + direction: DiagnosisDirection; +} { + const width = bitWidth(input.epsilon); + const experiments = [...input.experiments].sort((a, b) => compareIds(a.experiment, b.experiment)); + const rows = experiments.map(({ experiment, outcome }) => { + const entries = input.explanations.map((e) => ({ + id: e.id, + author: e.author, + range: e.experiments.find((x) => x.experiment === experiment)!.range, + })); + const mixture = ablationCellMixture(entries, input.lambda); + const valid = outcome.kind === "valid"; + return { + experiment, + outcome: outcome.kind, + gain: valid ? improvement(input.baseline, outcome.score, input.direction) : null, + merged: mixtureRange(mixture), + worldBits: valid ? worldBits(mixture, input.baseline, width) : null, + surprise: valid ? surprise(mixture, input.baseline, width, outcome.score) : null, + disagreement: valid ? disagreement(entries, input.baseline, width) : null, + }; + }); + // 确认与否定沿用消融的判法：把"实验"当成"格"交给 judgeExplanations，结果再换回实验的叫法。 + const asCells: AblationExplanation[] = input.explanations.map((e) => ({ + id: e.id, + author: e.author, + cells: e.experiments.map((x) => ({ factor: x.experiment, range: x.range })), + })); + const judgments = judgeExplanations(asCells, experiments.map((e) => ({ factor: e.experiment, outcome: e.outcome }))).map( + (j) => ({ id: j.id, verdicts: j.verdicts.map((v) => ({ experiment: v.factor, verdict: v.verdict })) }), + ); + return { + experiments: rows, + judgments, + direction: diagnosisDirection( + diagnosisModel({ ...input, experiments: experiments.map((e) => e.experiment) }), + experiments.map((e) => (e.outcome.kind === "valid" ? e.outcome.score : null)), + ), + }; +} + +/** 诊断推出的方向：按有效实验的实测更新成分权重，有效实验用实测换算收益，其余实验用新权重估，取收益最大的（6.12"结算"）。 */ +function diagnosisDirection(model: DiagnosisModel, scores: ReadonlyArray): DiagnosisDirection { + const { weights, kept } = updatedWeights(model, scores); + const gains = model.experiments.map((experiment, d) => { + const x = scores[d] ?? null; + return { experiment, gain: itemGain(model, d, weights, x), measured: x !== null }; + }); + let best = 0; + for (let d = 1; d < gains.length; d++) if (gains[d]!.gain > gains[best]!.gain) best = d; + return { + weights, + weightsKept: kept, + gains, + experiment: gains[best]!.experiment, + gain: gains[best]!.gain, + found: gains[best]!.gain > 0, + }; +} diff --git a/src/rules/index.ts b/src/rules/index.ts index 667041f..85bd734 100644 --- a/src/rules/index.ts +++ b/src/rules/index.ts @@ -103,6 +103,23 @@ export { worldBits, type Placement, } from "./bits.ts"; +export { + diagnosisModel, + evaluateDiagnosisCombos, + selectDiagnosis, + settleDiagnosis, + type DiagnosisCombo, + type DiagnosisDecision, + type DiagnosisDirection, + type DiagnosisExplanation, + type DiagnosisModel, + type DiagnosisModelInput, + type DiagnosisParams, + type DiagnosisRuleInput, + type ExperimentOutcome, + type SettleDiagnosisInput, + type SkippedReason, +} from "./diagnosis.ts"; export { ablationCellLimit, forwardBatchSize, runBalance, type RunState } from "./budget.ts"; export { authorVsMerged, trackRecord, type Phase, type Tally } from "./record.ts"; export { assignEntries, type EntryAssignment } from "./entries.ts"; @@ -110,8 +127,11 @@ export { decisionSeed, seededRandom, seededSample } from "./random.ts"; export { BATCH_RULES, CELL_RULES, + DIAGNOSIS_RULES, runBatchRule, runCellRule, + runDiagnosisRule, type BatchRuleName, type CellRuleName, + type DiagnosisRuleName, } from "./registry.ts"; diff --git a/src/rules/record.ts b/src/rules/record.ts index d698b74..64bfd68 100644 --- a/src/rules/record.ts +++ b/src/rules/record.ts @@ -2,10 +2,12 @@ import type { Direction, Range } from "../vocab/index.ts"; import { compareIds, improvement } from "./basics.ts"; import { covers, placement, type Placement } from "./bits.ts"; -// 方法.md 6.10 第 4 条"战绩"、6.9"按人分开、按正向与消融分开"。只用来显示，不改任何权重。 +// 方法.md 6.10 第 4 条"战绩"、6.9"按人分开、按正向、诊断与消融分开"。只用来显示，不改任何权重。 // 一律按分数原值比，不按方向折算。 -export type Phase = "forward" | "ablation"; +export type Phase = "forward" | "diagnosis" | "ablation"; + +const PHASE_ORDER: readonly Phase[] = ["forward", "diagnosis", "ablation"]; /** 一个人的战绩。 */ export interface Tally { @@ -25,8 +27,8 @@ export interface Tally { } /** - * 战绩：按研究者、按正向与消融分开数（6.9、6.10）。samples 是每人每次的范围（第二次预测或解释的每格）和有效实测。 - * 结果按研究者编号、再按 forward、ablation 排，只列有样本的组合。 + * 战绩：按研究者、按正向、诊断与消融分开数（6.9、6.10）。samples 是每人每次的范围（第二次预测、诊断预测或解释的每格）和有效实测。 + * 结果按研究者编号、再按 forward、diagnosis、ablation 排，只列有样本的组合。 */ export function trackRecord( samples: ReadonlyArray<{ researcher: string; phase: Phase; range: Range; value: number }>, @@ -62,7 +64,7 @@ export function trackRecord( t.placements[placement(r, v)] += 1; } return [...groups.values()].sort( - (a, b) => compareIds(a.researcher, b.researcher) || (a.phase === b.phase ? 0 : a.phase === "forward" ? -1 : 1), + (a, b) => compareIds(a.researcher, b.researcher) || PHASE_ORDER.indexOf(a.phase) - PHASE_ORDER.indexOf(b.phase), ); } diff --git a/src/rules/registry.ts b/src/rules/registry.ts index 76a6a55..f4ace51 100644 --- a/src/rules/registry.ts +++ b/src/rules/registry.ts @@ -1,4 +1,5 @@ import { selectAll, selectByValue, selectNone, selectRandomSameCount, type CellDecision, type CellRuleInput } from "./cells.ts"; +import { selectDiagnosis, type DiagnosisDecision, type DiagnosisRuleInput } from "./diagnosis.ts"; import { selectBestExpected, selectUniformRandom, type BatchDecision, type BatchInput } from "./forward.ts"; // 规则登记表（结构.md 第五部分"决策规则"、方法.md 第十一部分）：名字 → 函数，带版本号。 @@ -22,8 +23,14 @@ export const CELL_RULES = { none: { version: 1, run: selectNone }, } as const satisfies Record CellDecision }>; +export const DIAGNOSIS_RULES = { + /** 信息价值减成本（6.12）；选诊断实验只有这一条规则，配置里不选。 */ + valueMinusCost: { version: 1, run: selectDiagnosis }, +} as const satisfies Record DiagnosisDecision }>; + export type BatchRuleName = keyof typeof BATCH_RULES; export type CellRuleName = keyof typeof CELL_RULES; +export type DiagnosisRuleName = keyof typeof DIAGNOSIS_RULES; /** 按名字跑一条选批规则，结果带上规则名和版本。 */ export function runBatchRule( @@ -42,3 +49,12 @@ export function runCellRule( const entry = CELL_RULES[name]; return { rule: { name, version: entry.version }, ...entry.run(input) }; } + +/** 按名字跑一条选诊断实验规则，结果带上规则名和版本。 */ +export function runDiagnosisRule( + name: DiagnosisRuleName, + input: DiagnosisRuleInput, +): DiagnosisDecision & { rule: { name: DiagnosisRuleName; version: number } } { + const entry = DIAGNOSIS_RULES[name]; + return { rule: { name, version: entry.version }, ...entry.run(input) }; +} diff --git a/src/rules/value.ts b/src/rules/value.ts index 55506c4..97f8771 100644 --- a/src/rules/value.ts +++ b/src/rules/value.ts @@ -9,13 +9,44 @@ import { GRID_SUBDIVISIONS, improvement, } from "./basics.ts"; -import { ablationCellMixture, below, integrateUpperTail, knotsMean, mapKnots, type Knots } from "./distribution.ts"; +import { + ablationCellMixture, + below, + density, + integrateUpperTail, + knotsMean, + mapKnots, + type Knots, +} from "./distribution.ts"; // 方法.md 6.7：方向收益、成分与权重、信息价值、价值减成本。选格规则和押的因子在 cells.ts。 // -// 一律先在"撑几成"刻度 s(x) = [f(完整) − x] ÷ G（按方向折算）上算，最后乘剩余空间 h： -// 收益_p = h × s_p，h 对所有因子相同且为正，所以信息价值、期望收益、ĝ₀ 都等于 h × (撑几成刻度上的同一个量)。 -// 例外是部分格网格的切段：在分数原值上取折点、切小段，再把中点换成撑几成（见 measuredGrid）。 +// 成分、权重更新、精确积分、部分格网格、价值减成本这套引擎消融（6.7）和诊断（6.12）共用，只差"实测 → 收益"的换算： +// - 消融：一律先在"撑几成"刻度 s(x) = [f(完整) − x] ÷ G（按方向折算）上算，最后乘剩余空间 h： +// 收益_p = h × s_p，h 对所有因子相同且为正，所以信息价值、期望收益、ĝ₀ 都等于 h × (撑几成刻度上的同一个量)； +// 不设"都不走"的选项。 +// - 诊断（diagnosis.ts）：收益 g(x) = 改进量 x − b，h 取 1，另有收益为 0 的"都不走"，即 floor = 0。 +// 例外是部分格网格的切段：在分数原值上取折点、切小段，再把中点换成收益刻度（见 gridOf）。 + +/** 引擎要的成分模型（消融和诊断各自建出来）。下标"项"是因子（消融）或实验（诊断）。 */ +export interface ValueModel { + /** 成分起始权重。 */ + weights: number[]; + /** rawKnots[项][成分]：分数原值上的折线。 */ + rawKnots: Knots[][]; + /** scaledKnots[项][成分]：收益刻度上的折线。 */ + scaledKnots: Knots[][]; + /** expected[项][成分]：这个成分下这一项在收益刻度上的期望。 */ + expected: number[][]; + /** prior[项]：按起始权重平均的期望。 */ + prior: number[]; + /** 收益 = space × 收益刻度上的值：消融是剩余空间 h，诊断是 1。 */ + space: number; + /** "都不走"的收益：诊断是 0，消融没有这个选项，写 null。 */ + floor: number | null; + /** 分数原值 → 收益刻度。 */ + scale: (x: number) => number; +} /** 开放预留成分的名字（成分名单里排在最后）。 */ export const OPEN_RESERVE = "openReserve"; @@ -70,6 +101,20 @@ function supportScale(full: number, direction: Direction, gain: number): (x: num return (x) => improvement(x, full, direction) / gain; } +/** 消融模型在引擎眼里的样子：收益刻度是撑几成，没有"都不走"。 */ +export function ablationView(model: AblationModel): ValueModel { + return { + weights: model.weights, + rawKnots: model.rawKnots, + scaledKnots: model.supportKnots, + expected: model.expected, + prior: model.prior, + space: model.space, + floor: null, + scale: supportScale(model.full, model.direction, model.gain), + }; +} + /** * 成分模型（6.7"成分与权重"）：每条解释一个成分，开放预留一个成分；每个成分对每一格有一个分布， * 解释用它自己的三个数补尾巴，开放预留用这一格的均匀分布（6.3"消融"）。同一成分内各格互相独立。 @@ -144,26 +189,33 @@ export function cellGrid(knots: readonly Knots[], k: number): { mids: number[]; } /** - * 模型里第 c 格的网格（6.7"只测一部分格"），测 k 格时用。在分数原值上取折点、切小段：方法说"每格在自己的开放预留范围内 + * 模型里第 c 项的网格（6.7"只测一部分格"），测 k 项时用。在分数原值上取折点、切小段：方法说"每格在自己的开放预留范围内 * 按折点切段"，开放预留和折点都是在分数原值上定的（尾巴按 q1 − (q5 − q1)、q9 + (q9 − q5)，预留按 L − W/2、U + W/2 - * 的浮点原式），去重只合并浮点上完全相等的值。先换到撑几成刻度再去重，相差一个末位的两个原值折点可能换算后变成同一个数， - * 段数和 s 就会跟原值上的不一样。切好后只把小段中点换算成撑几成；把握与刻度无关。 + * 的浮点原式），去重只合并浮点上完全相等的值。先换到收益刻度再去重，相差一个末位的两个原值折点可能换算后变成同一个数， + * 段数和 s 就会跟原值上的不一样。切好后只把小段中点换算成收益刻度；把握与刻度无关。 */ -export function measuredGrid(model: AblationModel, c: number, k: number): { mids: number[]; mass: number[][] } { +export function gridOf(model: ValueModel, c: number, k: number): { mids: number[]; mass: number[][] } { const { mids, mass } = cellGrid(model.rawKnots[c]!, k); - return { mids: mids.map(supportScale(model.full, model.direction, model.gain)), mass }; + return { mids: mids.map(model.scale), mass }; +} + +/** 消融模型里第 c 格的网格，中点换算成撑几成（6.7"只测一部分格"）。 */ +export function measuredGrid(model: AblationModel, c: number, k: number): { mids: number[]; mass: number[][] } { + return gridOf(ablationView(model), c, k); } /** - * 只测 S 里的格时的 E_{S 的结果}[max_p E(撑_p | S 的结果)]（6.7"信息价值"），在撑几成刻度上。 - * 对测到的格的全部小段组合求和：成分 j 的联合把握 = 起始权重 × 各测到格在所选小段的把握；它们的和 marg 是这个组合的把握， - * 按它归一就是更新后的权重。网格见 measuredGrid。测到的格取小段中点（已换成撑几成），没测的格用更新后的权重平均各成分的期望，取各因子里最大的。 + * 只测 S 里的项时的 E_{S 的结果}[max(floor, max_p E(收益_p | S 的结果))]（6.7"信息价值"、6.12），在收益刻度上， + * 没有"都不走"（floor 为 null）时不和 floor 取大。 + * 对测到的项的全部小段组合求和：成分 j 的联合把握 = 起始权重 × 各测到项在所选小段的把握；它们的和 marg 是这个组合的把握， + * 按它归一就是更新后的权重。网格见 gridOf。测到的项取小段中点（已换成收益刻度），没测的项用更新后的权重平均各成分的期望，取各项里最大的。 */ -function expectedBestGrid(model: AblationModel, S: readonly number[]): number { +function expectedBestGrid(model: ValueModel, S: readonly number[]): number { const k = S.length; const J = model.weights.length; - const grids = S.map((c) => measuredGrid(model, c, k)); - const rest = model.factors.map((_, c) => c).filter((c) => !S.includes(c)); + const floor = model.floor ?? -Infinity; + const grids = S.map((c) => gridOf(model, c, k)); + const rest = model.prior.map((_, c) => c).filter((c) => !S.includes(c)); const partial = Array.from({ length: k + 1 }, () => new Float64Array(J)); for (let j = 0; j < J; j++) partial[0]![j] = model.weights[j]!; let total = 0; @@ -173,7 +225,7 @@ function expectedBestGrid(model: AblationModel, S: readonly number[]): number { let marg = 0; for (let j = 0; j < J; j++) marg += cur[j]!; if (marg === 0) return; - let best = seen; + let best = Math.max(seen, floor); for (const c of rest) { const e = model.expected[c]!; let num = 0; @@ -195,36 +247,80 @@ function expectedBestGrid(model: AblationModel, S: readonly number[]): number { } /** - * 全部格都测时的 E[max_p 撑_p]（6.7"信息价值"）：同一成分内"各格撑几成的最大值低于 t"的把握 = 各格把握连乘， - * 期望 = 下端 + ∫(1 − 连乘) dt，逐段多项式精确积分；再按起始权重相加。 + * 全部项都测时的 E[max(floor, max_p 收益_p)]（6.7"信息价值"、6.12）：同一成分内"各项最大值低于 t"的把握 = 各项把握连乘， + * 期望 = 下端 + ∫(1 − 连乘) dt，逐段多项式精确积分；再按起始权重相加。下端：有"都不走"时是 floor（t 低于它时 + * max(floor, ·) 一定不低于 t，被积函数是 1），没有时是各项折线的最低点。 */ -function expectedBestAll(model: AblationModel): number { +function expectedBestAll(model: ValueModel): number { let v = 0; for (let j = 0; j < model.weights.length; j++) { - const lines = model.supportKnots.map((cell) => cell[j]!); - const lo = Math.min(...lines.map((k) => k[0]![0])); + const lines = model.scaledKnots.map((cell) => cell[j]!); + const lo = model.floor ?? Math.min(...lines.map((k) => k[0]![0])); v += model.weights[j]! * (lo + integrateUpperTail(lines.map((knots) => [{ weight: 1, knots }]), lo)); } return v; } +/** max(floor, max_p E(收益_p))：不测时能拿到的最好期望，在收益刻度上。 */ +function bestPrior(model: ValueModel): number { + return Math.max(model.floor ?? -Infinity, ...model.prior); +} + +/** 用网格算的信息价值（不论测几项都走网格），S 是项的下标。 */ +export function valueByGrid(model: ValueModel, S: readonly number[]): number { + return model.space * (expectedBestGrid(model, [...S].sort((a, b) => a - b)) - bestPrior(model)); +} + +/** 全部项都测时的信息价值（一维精确积分）。 */ +export function valueExact(model: ValueModel): number { + return model.space * (expectedBestAll(model) - bestPrior(model)); +} + +/** + * 信息价值 VOI(S) = E_{S 的结果}[max(floor, max_p E(收益_p | S 的结果))] − max(floor, max_p E(收益_p))（6.7、6.12）， + * S 是项的下标；全部项都测走一维精确积分，只测一部分走网格，空集是 0。不截断。 + */ +export function valueOf(model: ValueModel, S: readonly number[]): number { + if (S.length === 0) return 0; + return S.length === model.prior.length ? valueExact(model) : valueByGrid(model, S); +} + /** 网格算法算的信息价值，S 是格的下标（不论测几格都走网格；用来和精确算法互相对照）。 */ export function informationValueGrid(model: AblationModel, S: readonly number[]): number { - return model.space * (expectedBestGrid(model, [...S].sort((a, b) => a - b)) - Math.max(...model.prior)); + return valueByGrid(ablationView(model), S); } /** 全部格都测时的信息价值（一维精确积分）。 */ export function informationValueExact(model: AblationModel): number { - return model.space * (expectedBestAll(model) - Math.max(...model.prior)); + return valueExact(ablationView(model)); +} + +/** 消融的信息价值 VOI(S)（6.7），S 是格的下标。 */ +export function informationValue(model: AblationModel, S: readonly number[]): number { + return valueOf(ablationView(model), S); } /** - * 信息价值 VOI(S) = E_{S 的结果}[max_p E(收益_p | S 的结果)] − max_p E(收益_p)（6.7），S 是格的下标； - * 全部格都测走一维精确积分，只测一部分走网格，空集是 0。不截断。 + * 成分新权重（6.8、6.12）：起始权重 × 各有效项（scores 里不是 null 的）上这个成分在实测处的密度（分数原值上，连乘）， + * 再归一；实测落在所有成分之外、权重之和为 0 时保持起始权重。scores 与项的下标对齐，没有有效实测的项为 null。 */ -export function informationValue(model: AblationModel, S: readonly number[]): number { - if (S.length === 0) return 0; - return S.length === model.factors.length ? informationValueExact(model) : informationValueGrid(model, S); +export function updatedWeights( + model: ValueModel, + scores: ReadonlyArray, +): { weights: number[]; kept: boolean } { + const raw = model.weights.map((w, j) => + scores.reduce((v, x, c) => (x === null ? v : v * density(model.rawKnots[c]![j]!, x)), w), + ); + const sum = raw.reduce((t, v) => t + v, 0); + const kept = sum === 0; + return { weights: kept ? [...model.weights] : raw.map((v) => v / sum), kept }; +} + +/** 第 c 项的方向收益（6.8、6.12）：有有效实测 x 就用实测换算，否则用 weights 对各成分的期望加权平均。 */ +export function itemGain(model: ValueModel, c: number, weights: readonly number[], x: number | null): number { + return x !== null + ? model.space * model.scale(x) + : model.space * model.expected[c]!.reduce((t, e, j) => t + weights[j]! * e, 0); } /** 一个测格组合的价值与成本（6.7"价值减成本"）。 */ @@ -253,6 +349,19 @@ export interface ComboParams { decay: number; } +/** m′ = 1 + r + … + r^(R̂−1)，R̂ = 0 时是 0（6.7"价值减成本"）；消融和诊断共用。 */ +export function usePotential(rHat: number, decay: number): number { + let mPrime = 0; + for (let j = 0; j < rHat; j++) mPrime += decay ** j; + return mPrime; +} + +/** 成本 = 2 max(0, ĝ₀) × [(K + n)/(K + n + 1) − K/(K + 1)]，n 是这次占用的运行次数（6.7、6.12）；消融和诊断共用。 */ +export function runCost(g0: number, batchCap: number, runs: number): number { + const K = batchCap; + return 2 * Math.max(0, g0) * ((K + runs) / (K + runs + 1) - K / (K + 1)); +} + /** * 列出格数 1..min(格数, 余额 − 1) 的全部非空组合并算价值、成本、净值（6.7"选格"第 1 条）。余额为 0 或 1 时一个都没有。 * 组合按格数从少到多、同格数按因子编号字典序列出。ĝ₀ = h × max_p 期望撑几成，原值另给，成本里取 max(0, ĝ₀)。 @@ -265,10 +374,9 @@ export function evaluateCombos(model: AblationModel, params: ComboParams): { g0: for (const S of combinations(model.factors.length, size)) { const voi = informationValue(model, S); const rHat = Math.ceil((params.balance - size) / K); - let mPrime = 0; - for (let j = 0; j < rHat; j++) mPrime += params.decay ** j; + const mPrime = usePotential(rHat, params.decay); const value = params.u * mPrime * voi; - const cost = 2 * Math.max(0, g0) * ((K + size) / (K + size + 1) - K / (K + 1)); + const cost = runCost(g0, K, size); combos.push({ factors: S.map((c) => model.factors[c]!), voi, rHat, mPrime, value, cost, net: value - cost }); } } diff --git a/src/submissions/checks.ts b/src/submissions/checks.ts index e1d8a1b..28d51a0 100644 --- a/src/submissions/checks.ts +++ b/src/submissions/checks.ts @@ -23,6 +23,15 @@ export function programProblems(inc: Incoming, protocol: ProtocolBody | null): s ...factorProblems(inc.content, inc.checklist), ...increasing("authorEstimate", inc.content.authorEstimate), ]; + case "diagnosisProposal": + return diagnosisProposalProblems(inc.content, inc.checklist); + case "diagnosisPrediction": + return diagnosisPredictionProblems(inc.content, inc.checklist); + case "diagnosisImplementation": + case "diagnosisRepair": + return codeProblems(inc.known.code); + case "diagnosisVerification": + return covers("experiments 的实验", inc.content.experiments.map((x) => x.experiment), inc.checklist.experiments); case "firstPrediction": case "secondPrediction": return increasing("range", inc.content.range); @@ -63,6 +72,45 @@ function claimProblems(c: Submission<"claim">, k: ChecklistOf<"claim">): string[ ]; } +/** + * 诊断解释与诊断实验（方法.md 4.3 诊 1、第五部分"指标"）：解释 1 至 M 条、实验 1 至 A 个（下限由字段定义管）； + * 诊断实验登记的指标只能选"缺了只记缺失"；同一个实验里指标短名不重名、不占用已有编号 + * （不同实验的同名短名冻结时各带自己实验的前缀，不会撞）。 + */ +function diagnosisProposalProblems(c: Submission<"diagnosisProposal">, k: ChecklistOf<"diagnosisProposal">): string[] { + return [ + ...(c.explanations.length > k.maxExplanations ? [`交了 ${c.explanations.length} 条诊断解释，最多 ${k.maxExplanations} 条`] : []), + ...(c.experiments.length > k.maxExperiments ? [`交了 ${c.experiments.length} 个诊断实验，最多 ${k.maxExperiments} 个`] : []), + ...c.experiments.flatMap((e, i) => { + const ids = e.metrics.map((m) => m.id); + return [ + ...e.metrics + .filter((m) => m.missing !== "recordMissing") + .map((m) => `第 ${i + 1} 个实验的指标 ${m.id} 选了 ${m.missing}：诊断实验里的指标只能选 recordMissing（缺了只记缺失）`), + ...repeated(ids).map((x) => `第 ${i + 1} 个实验的 metrics 里编号 ${x} 写了不止一次`), + ...unique(ids) + .filter((x) => k.takenMetricIds.includes(x)) + .map((x) => `第 ${i + 1} 个实验的指标编号 ${x} 已被占用（已占用的是 ${k.takenMetricIds.join("、")}）：换一个编号`), + ]; + }), + ]; +} + +/** + * 诊断预测（方法.md 4.3 诊 2）：覆盖自己每条解释 × 每个合并后的实验，不多不少、每一对一次；每个范围三个数严格递增。 + * 一对写成"解释编号:实验编号"（编号里没有冒号）。 + */ +function diagnosisPredictionProblems(c: Submission<"diagnosisPrediction">, k: ChecklistOf<"diagnosisPrediction">): string[] { + return [ + ...covers( + "predictions 的（解释:实验）", + c.predictions.map((p) => `${p.explanation}:${p.experiment}`), + k.explanations.flatMap((e) => k.experiments.map((d) => `${e}:${d}`)), + ), + ...c.predictions.flatMap((p) => increasing(`${p.explanation}:${p.experiment} 的 range`, p.range)), + ]; +} + /** * 提案、回应、统一共用（方法.md 第五部分"延续""指标"、4.3 ②⑥）： * 规则 2 因子数上限；3 两个因子不延续同一个、延续的编号看得到；4 保障短名；10 指标的引用与新登记。 diff --git a/src/submissions/index.ts b/src/submissions/index.ts index b0df3e3..12b8168 100644 --- a/src/submissions/index.ts +++ b/src/submissions/index.ts @@ -1,6 +1,6 @@ // submissions：一份提交收下还是退回，先程序检查、再问 Jev（docs/结构.md 第二部分）； // 另有冻结协议的拼法，和核验、消融核验收下之后通过没有。不读也不写账本。 -export { ablationOutcome, verificationOutcome } from "./outcomes.ts"; +export { ablationOutcome, diagnosisOutcome, verificationOutcome } from "./outcomes.ts"; export { protocolBody } from "./protocol.ts"; export { receive } from "./receive.ts"; export { diff --git a/src/submissions/outcomes.ts b/src/submissions/outcomes.ts index dba5c24..2efa746 100644 --- a/src/submissions/outcomes.ts +++ b/src/submissions/outcomes.ts @@ -34,3 +34,20 @@ export function ablationOutcome( const wronglyOff = v.factors.filter((f) => !shouldBeOff.has(f.factor) && f.off).map((f) => f.factor); return { passed: leftOn.length + wronglyOff.length === 0, leftOn, wronglyOff }; } + +/** + * 诊断核验通过没有（方法.md 4.3 诊 4）：逐个实验判，任何一项写了 false 的实验不通过。 + * notFollowed 是没照写法做的实验，touchedElse 是动了写法以外的地方的实验，failing 是两者的并集，按核验里的次序。 + * 一个实验不通过不影响别的实验：通过的实验代码不再改动。 + */ +export function diagnosisOutcome(v: Submission<"diagnosisVerification">): { + passed: boolean; + failing: string[]; + notFollowed: string[]; + touchedElse: string[]; +} { + const notFollowed = v.experiments.filter((x) => !x.followed).map((x) => x.experiment); + const touchedElse = v.experiments.filter((x) => !x.nothingElse).map((x) => x.experiment); + const failing = v.experiments.filter((x) => !x.followed || !x.nothingElse).map((x) => x.experiment); + return { passed: failing.length === 0, failing, notFollowed, touchedElse }; +} diff --git a/src/submissions/receive.ts b/src/submissions/receive.ts index 001cc1a..e75fcb1 100644 --- a/src/submissions/receive.ts +++ b/src/submissions/receive.ts @@ -1,4 +1,4 @@ -import { checkHunk, checkProtocol, type CodeHunk } from "../jev/index.ts"; +import { checkExperiment, checkHunk, checkProtocol, type CodeHunk } from "../jev/index.ts"; import type { JevJudgment } from "../vocab/index.ts"; import { programProblems } from "./checks.ts"; import { protocolBody } from "./protocol.ts"; @@ -27,7 +27,8 @@ export async function receive(inc: Incoming): Promise { } /** - * 问 Jev（方法.md 4.3 ⑥⑨）：回应、统一对拼好的协议问冻结条款三问；实现、修复每个差异块问代码改动五问，顺序发。 + * 问 Jev（方法.md 4.3 ⑥⑨、诊 4）：回应、统一对拼好的协议问冻结条款三问；实现、修复（含诊断程序）每个差异块问代码改动五问，顺序发。 + * 诊断解释与实验（诊 1）对每个实验问冻结条款三问（改动了评测、自相矛盾、太含糊，用诊断的变体问法），顺序发。 * 没有差异块的改动（二进制、只改可执行位、空文件增删）Jev 看不到内容，不问；一块都没有就等于没问，交回 null。 * 消融实现不问：冻结协议里没有登记"关掉以后"的随机性和停止条件，五问没有对照（照参考项目）。 * flagged 是退回原因里每一处的文字：只列判"是"的几问，问题名用 jev 的英文名（用户 2026-10-03 定）；判"是"只看 jev 交回的 yes。 @@ -37,7 +38,19 @@ async function askJev(inc: Incoming, protocol: ProtocolBody | null): Promise<{ j const c = await checkProtocol(inc.jevModel, protocol!); return { judgments: c.judgments, flagged: c.flagged ? [`- 冻结协议：${flaggedText(c.judgments)}`] : [] }; } - if (inc.kind !== "implementation" && inc.kind !== "repair") return null; + if (inc.kind === "diagnosisProposal") { + const judgments: JevJudgment[] = []; + const flagged: string[] = []; + for (const [i, experiment] of inc.content.experiments.entries()) { + const c = await checkExperiment(inc.jevModel, experiment); + judgments.push(...c.judgments); + if (c.flagged) flagged.push(`- 第 ${i + 1} 个诊断实验（${experiment.name}）：${flaggedText(c.judgments)}`); + } + return { judgments, flagged }; + } + if (inc.kind !== "implementation" && inc.kind !== "repair" && inc.kind !== "diagnosisImplementation" && inc.kind !== "diagnosisRepair") { + return null; + } // 走到这里程序检查已过，代码一定存成了版本。 const code = inc.known.code as Extract; if (code.hunks.length === 0) return null; diff --git a/src/submissions/types.ts b/src/submissions/types.ts index c519c8e..ab0bade 100644 --- a/src/submissions/types.ts +++ b/src/submissions/types.ts @@ -17,7 +17,7 @@ export type OriginalProposal = Pick, "factors" | "safegua type NotSaved = { saved: false; path: string; reason: string }; /** - * 实现、修复的代码：存成了版本就带改了的不许改的文件（tasks 的 protectedChanges）、 + * 实现、修复（候选和诊断程序同样）的代码：存成了版本就带改了的不许改的文件（tasks 的 protectedChanges）、 * 和本批基线比出的差异块（jev 的 codeHunks 切好的），以及给 Jev 对照的协议正文和要采集的指标。 */ export type ForwardCode = NotSaved | { saved: true; protectedFiles: string[]; hunks: CodeHunk[]; protocol: ProtocolRules }; @@ -33,6 +33,9 @@ type KnownByKind = { response: { original: OriginalProposal }; implementation: { code: ForwardCode }; repair: { code: ForwardCode }; + /** 诊断程序和候选代码同形：和当前基线比出的差异块，给 Jev 对照的是选中实验的原文和要采集的指标。 */ + diagnosisImplementation: { code: ForwardCode }; + diagnosisRepair: { code: ForwardCode }; /** 交了"跑不起来"（cannotRun 不是 null）时 flow 不存代码，code 为 null。 */ ablationImplementation: { code: AblationCode | null }; }; diff --git a/src/tasks/circle-packing.ts b/src/tasks/circle-packing.ts index 5cc4b96..50b13bd 100644 --- a/src/tasks/circle-packing.ts +++ b/src/tasks/circle-packing.ts @@ -1,4 +1,4 @@ -import { copyFileSync, cpSync, existsSync, readFileSync, writeFileSync } from "node:fs"; +import { chmodSync, copyFileSync, cpSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { Type, type Static } from "typebox"; import { Value } from "typebox/value"; @@ -13,7 +13,16 @@ import type { } from "../vocab/index.ts"; import type { TaskFacts } from "./facts.ts"; import { measured, readMetricsFile, unmeasured } from "./metrics-file.ts"; -import type { LightCheckOutcome, Task, TaskImplementation } from "./types.ts"; +import type { + DiagnosisEnding, + DiagnosisExperiment, + DiagnosisExperimentJudged, + DiagnosisJudged, + DiagnosisJudgingStep, + LightCheckOutcome, + Task, + TaskImplementation, +} from "./types.ts"; // 填圆：在单位正方形里放 26 个圆，最大化半径之和。任务说明见 tasks/circle_packing/TASK.md。 // 凡是要执行任务代码的事（正式运行、校验器、轻量检查）都只交出运行定义，由调用方交给 runner 在任务镜像里跑； @@ -22,6 +31,7 @@ import type { LightCheckOutcome, Task, TaskImplementation } from "./types.ts"; const CIRCLES = 26; const ENTRY = "initial.py"; const HARNESS = "harness.py"; +const DIAGNOSIS_HARNESS = "harness-diagnosis.py"; const VALIDATOR = "validate.py"; const CHECK_SCRIPT = "check-candidate.py"; const METRICS_SPEC = "metrics-spec.json"; @@ -29,6 +39,12 @@ const METRICS = "metrics.json"; const SOLUTION = "solution.json"; const VERDICT = "verdict.json"; const CHECK_RESULT = "check.json"; +/** 诊断运行：原始产物目录、每个实验的答案目录、harness 写的每实验用时报告（路径都相对运行目录）。 */ +const BASELINE_ARTIFACT = "baseline-artifact"; +const ANSWERS = "answers"; +const DIAGNOSIS_REPORT = "diagnosis.json"; +/** 判分目录里每个实验写的校验器退出码文件（同一个目录下还有这个实验的 solution.json、verdict.json）。 */ +const EXIT_FILE = "exit"; /** 轻量检查目录里放候选代码的子目录；检查脚本放在它的上一层，免得扫到自己。 */ const CHECK_CANDIDATE = "candidate"; /** 运行目录挂进容器的位置。 */ @@ -277,16 +293,16 @@ function readVerdict(judgeDir: string, exitCode: number): ValidatorVerdict { return verdict; } -function judge( - _task: Task, - runDir: string, - judgeDir: string, - formal: ProcessResult, - validator: ProcessResult, - hypothesisMetrics: HypothesisMetric[], -): JudgedResult { - const run = started(formal); - const verdict = readVerdict(judgeDir, requireExit("校验器", validator, [0, 1])); +/** 校验器判定读成的结论：算不算数、理由、最终分数、校验器给的辅助指标、合法化明细。 */ +interface Scored { + valid: boolean; + reason: string; + score: Measured; + fromValidator: Record; + extra: Json; +} + +function scoreVerdict(verdict: ValidatorVerdict): Scored { const legal = verdict.legalization; let deduction: Measured; let score: Measured; @@ -328,7 +344,199 @@ function judge( deduction, }, }; - return assemble(runDir, run, hypothesisMetrics, { valid: verdict.valid, reason }, score, fromValidator, extra); + return { valid: verdict.valid, reason, score, fromValidator, extra }; +} + +function judge( + _task: Task, + runDir: string, + judgeDir: string, + formal: ProcessResult, + validator: ProcessResult, + hypothesisMetrics: HypothesisMetric[], +): JudgedResult { + const run = started(formal); + const scored = scoreVerdict(readVerdict(judgeDir, requireExit("校验器", validator, [0, 1]))); + return assemble(runDir, run, hypothesisMetrics, { valid: scored.valid, reason: scored.reason }, scored.score, scored.fromValidator, scored.extra); +} + +// --------------------------------------------------------------------------- +// 诊断运行（方法.md 4.3"诊断"一段）：一个容器、一次登记、一次扣次，里面按顺序跑几个实验。 +// 全部是新增的一套函数，用自己的 harness（harness-diagnosis.py），不碰上面正式运行的任何一步。 +// --------------------------------------------------------------------------- + +/** 诊断程序存版本时跳过的名字：正式运行那份加上原始产物目录（它是宿主放进工作目录的，不是实验员的代码）。 */ +function diagnosisUnversionedNames(facts: TaskFacts): string[] { + return [...facts.unversionedNames, BASELINE_ARTIFACT]; +} + +/** + * 把产出当前基线的那次正式运行的原始产物（solution.json）放进 dir 下的 baseline-artifact/，设成只读。 + * 实验员的工作目录和诊断运行的运行目录都用它：实验员能看到实验要接着算的那份答案，运行时实验读的是同一份。 + * 只读靠文件权限，防的是实验顺手改了它，不是安全边界。 + */ +function placeBaselineArtifact(_task: Task, baselineRunDir: string, dir: string): void { + const target = join(dir, BASELINE_ARTIFACT); + mkdirSync(target); + const file = join(target, SOLUTION); + copyFileSync(join(baselineRunDir, SOLUTION), file); + chmodSync(file, 0o444); +} + +function prepareDiagnosisRun(task: Task, programDir: string, baselineRunDir: string, runDir: string, experiments: DiagnosisExperiment[]): void { + cpSync(programDir, runDir, { recursive: true }); + // 两份 harness 以宿主的为准；指标说明是全部实验登记的指标合在一起，每个实验只写自己的那几条。 + copyFileSync(join(task.dir, HARNESS), join(runDir, HARNESS)); + copyFileSync(join(task.dir, DIAGNOSIS_HARNESS), join(runDir, DIAGNOSIS_HARNESS)); + writeFileSync(join(runDir, METRICS_SPEC), `${JSON.stringify(experiments.flatMap((e) => e.metrics), null, 2)}\n`); + placeBaselineArtifact(task, baselineRunDir, runDir); + mkdirSync(join(runDir, ANSWERS)); +} + +/** 总时限、核数照任务规定，实验按顺序跑，每个实验分到总时限 ÷ 实验数（由 harness 执行）。 */ +function diagnosisRun(task: Task, imageDigest: string, seed: string, experimentIds: string[]): RunDefinition { + const eachSeconds = task.facts.run.wallSeconds / experimentIds.length; + const command = [ + "python", + "-B", + `${WORKDIR}/${DIAGNOSIS_HARNESS}`, + "--program", + ENTRY, + "--baseline", + BASELINE_ARTIFACT, + "--answers", + ANSWERS, + "--seconds-each", + String(eachSeconds), + "--report", + DIAGNOSIS_REPORT, + "--experiments", + ...experimentIds, + ]; + return definition(task, imageDigest, command, task.facts.run.wallSeconds, { INFORSI_SEED: seed }); +} + +/** 一个实验在运行目录里的结局：harness 报告里的一项，或者整次运行先结束、没轮到它。 */ +type ExperimentOutcome = { ending: "finished" | "crashed" | "timeout"; wallSeconds: number } | { ending: "notRun"; reason: string }; + +/** harness 写的报告（我们自己的程序，按约定的格式直接读）。 */ +interface ReportEntry { + id: string; + status: "finished" | "crashed" | "timeout"; + wall_seconds: number; +} + +function experimentOutcomes( + task: Task, + runDir: string, + process: Exclude, + experiments: DiagnosisExperiment[], +): ExperimentOutcome[] { + const path = join(runDir, DIAGNOSIS_REPORT); + const report = existsSync(path) ? (readJson(path) as ReportEntry[]) : []; + return experiments.map((e) => { + const entry = report.find((r) => r.id === e.id); + if (entry !== undefined) return { ending: entry.status, wallSeconds: entry.wall_seconds }; + const failure = processFailure(task, process); + if (failure === null) throw new Error(`诊断运行正常退出，但 ${DIAGNOSIS_REPORT} 里没有实验 ${e.id}`); + return { ending: "notRun", reason: `没轮到：整次诊断运行${failure}` }; + }); +} + +/** 没有答案可判的实验：不算数，辅助指标和假设专用指标都记测不到并附原因。 */ +function unjudgedExperiment( + id: string, + ending: Exclude, + wall: Measured, + reason: string, + hypothesisMetrics: HypothesisMetric[], +): DiagnosisExperimentJudged { + const metrics = { + ...Object.fromEntries(FROM_VALIDATOR.map((m) => [m, unmeasured(`没有跑校验器：${reason}`)])), + ...Object.fromEntries(hypothesisMetrics.map((m) => [m.id, unmeasured(`没有结果：${reason}`)])), + }; + return { id, ending, wallSeconds: wall, verdict: "invalid", verdictReason: reason, score: unmeasured(reason), metrics, extra: { legalization: null } }; +} + +function judgedExperiments( + task: Task, + runDir: string, + judgeDir: string, + experiments: DiagnosisExperiment[], + outcomes: ExperimentOutcome[], +): DiagnosisJudged { + const eachSeconds = task.facts.run.wallSeconds / experiments.length; + return { + experiments: experiments.map((e, i): DiagnosisExperimentJudged => { + const outcome = outcomes[i]!; + if (outcome.ending === "notRun") return unjudgedExperiment(e.id, "notRun", unmeasured(outcome.reason), outcome.reason, e.metrics); + const wall = measured(outcome.wallSeconds); + if (outcome.ending === "timeout") { + return unjudgedExperiment(e.id, "timeout", wall, `超过分到的时间 ${eachSeconds} 秒，被结束`, e.metrics); + } + if (outcome.ending === "crashed") { + return unjudgedExperiment(e.id, "crashed", wall, `实验的进程以非零退出码结束（报错见这次运行的 stderr.log）`, e.metrics); + } + const dir = join(judgeDir, e.id); + const exitCode = Number(readFileSync(join(dir, EXIT_FILE), "utf8").trim()); + const scored = scoreVerdict(readVerdict(dir, exitCode)); + const hypothesis = readMetricsFile(join(runDir, ANSWERS, e.id, METRICS), `${ANSWERS}/${e.id}/${METRICS}`, e.metrics).hypothesis; + return { + id: e.id, + ending: "finished", + wallSeconds: wall, + verdict: scored.valid ? "valid" : "invalid", + verdictReason: scored.reason, + score: scored.score, + metrics: { ...scored.fromValidator, ...hypothesis }, + extra: scored.extra, + }; + }), + }; +} + +/** 逐个实验判分。校验器一个容器、按顺序对每个有答案的实验各跑一次，退出码各存一个文件。 */ +const VALIDATE_EACH = `for id in "$@"; do +python -B ${VALIDATOR} --circles ${CIRCLES} --solution "$id/${SOLUTION}" --atol ${TOLERANCE} --legalize --verdict "$id/${VERDICT}" +echo $? > "$id/${EXIT_FILE}" +done`; + +function prepareDiagnosisJudging( + task: Task, + runDir: string, + judgeDir: string, + imageDigest: string, + process: ProcessResult, + experiments: DiagnosisExperiment[], +): DiagnosisJudgingStep { + const run = started(process); + const outcomes = experimentOutcomes(task, runDir, run, experiments); + const finished = experiments.filter((_, i) => outcomes[i]!.ending === "finished"); + if (finished.length === 0) return { status: "judged", result: judgedExperiments(task, runDir, judgeDir, experiments, outcomes) }; + for (const e of finished) { + mkdirSync(join(judgeDir, e.id)); + copyFileSync(join(runDir, ANSWERS, e.id, SOLUTION), join(judgeDir, e.id, SOLUTION)); + } + copyFileSync(join(task.dir, VALIDATOR), join(judgeDir, VALIDATOR)); + const command = ["sh", "-c", VALIDATE_EACH, "inforsi-validate-each", ...finished.map((e) => e.id)]; + return { status: "validate", definition: definition(task, imageDigest, command, VALIDATOR_SECONDS * finished.length, {}) }; +} + +function judgeDiagnosis( + task: Task, + runDir: string, + judgeDir: string, + process: ProcessResult, + validator: ProcessResult | null, + experiments: DiagnosisExperiment[], +): DiagnosisJudged { + const run = started(process); + const outcomes = experimentOutcomes(task, runDir, run, experiments); + if (outcomes.some((o) => o.ending === "finished")) { + if (validator === null) throw new Error("有实验交出了答案，却没有校验器的运行结果"); + requireExit("校验器", validator, [0]); + } + return judgedExperiments(task, runDir, judgeDir, experiments, outcomes); } export const circlePacking: TaskImplementation = { @@ -339,4 +547,10 @@ export const circlePacking: TaskImplementation = { readLightCheck, prepareJudging, judge, + diagnosisUnversionedNames, + placeBaselineArtifact, + prepareDiagnosisRun, + diagnosisRun, + prepareDiagnosisJudging, + judgeDiagnosis, }; diff --git a/src/tasks/index.ts b/src/tasks/index.ts index a1fbc6e..6828ab5 100644 --- a/src/tasks/index.ts +++ b/src/tasks/index.ts @@ -2,14 +2,29 @@ // 不起子进程：要执行任务代码的事都只交出运行定义，由调用方交给 runner 在任务镜像里跑。 export { checkTaskFacts, TASK_KINDS, TaskFacts, type TaskKind } from "./facts.ts"; export { + diagnosisRun, + diagnosisUnversionedNames, formalRun, judge, + judgeDiagnosis, lightCheck, loadTask, metricDefinitions, + placeBaselineArtifact, + prepareDiagnosisJudging, + prepareDiagnosisRun, prepareJudging, prepareRun, protectedChanges, readLightCheck, } from "./task.ts"; -export type { JudgingStep, LightCheckOutcome, Task } from "./types.ts"; +export type { + DiagnosisEnding, + DiagnosisExperiment, + DiagnosisExperimentJudged, + DiagnosisJudged, + DiagnosisJudgingStep, + JudgingStep, + LightCheckOutcome, + Task, +} from "./types.ts"; diff --git a/src/tasks/task.ts b/src/tasks/task.ts index 8551354..fe32887 100644 --- a/src/tasks/task.ts +++ b/src/tasks/task.ts @@ -3,7 +3,15 @@ import { join, resolve } from "node:path"; import type { HypothesisMetric, JudgedResult, MetricDefinition, ProcessResult, RunDefinition } from "../vocab/index.ts"; import { circlePacking } from "./circle-packing.ts"; import { checkTaskFacts, type TaskKind } from "./facts.ts"; -import type { JudgingStep, LightCheckOutcome, Task, TaskImplementation } from "./types.ts"; +import type { + DiagnosisExperiment, + DiagnosisJudged, + DiagnosisJudgingStep, + JudgingStep, + LightCheckOutcome, + Task, + TaskImplementation, +} from "./types.ts"; // tasks 对外的函数。任务目录的约定：task.json（任务事实）、TASK.md（给实验员的说明）、 // Dockerfile 与依赖清单（建镜像）、baseline/（基线代码），以及各任务自己的 Python 工具。 @@ -105,3 +113,74 @@ export function judge( export function protectedChanges(task: Task, changedFiles: string[]): string[] { return changedFiles.filter((file) => task.facts.protectedFiles.includes(file)); } + +// --------------------------------------------------------------------------- +// 诊断运行（方法.md 4.3"诊断"一段）：一个容器、一次登记、一次扣次，里面按顺序跑几个实验。 +// 是和上面正式运行并列的另一套函数，上面的函数不认识它。各目录同样由调用方建好、传空目录进来。 +// --------------------------------------------------------------------------- + +/** 存诊断程序版本时跳过的名字：task.json 的清单，加上宿主放进工作目录的原始产物目录。 */ +export function diagnosisUnversionedNames(task: Task): string[] { + return implementation(task).diagnosisUnversionedNames(task.facts); +} + +/** + * 把产出当前基线的那次正式运行的原始产物，只读地放进 dir（实验员工作目录，调用方已用 checkoutVersion 取出基线代码）。 + * baselineRunDir 是那次运行的运行目录。放在哪、叫什么，由任务定，写在 TASK.md 里给实验员看。 + */ +export function placeBaselineArtifact(task: Task, baselineRunDir: string, dir: string): void { + implementation(task).placeBaselineArtifact(task, baselineRunDir, dir); +} + +/** + * 准备一次诊断运行的运行目录：诊断程序、两份 harness、全部实验登记的指标说明（metrics-spec.json）、 + * 只读的原始产物。runDir 由调用方建好，应为空目录；实验的先后顺序就是 experiments 的顺序。 + */ +export function prepareDiagnosisRun( + task: Task, + programDir: string, + baselineRunDir: string, + runDir: string, + experiments: DiagnosisExperiment[], +): void { + implementation(task).prepareDiagnosisRun(task, programDir, baselineRunDir, runDir, experiments); +} + +/** + * 诊断运行的运行定义：总时限、核数照任务规定，harness 按顺序跑 experimentIds 里的实验， + * 每个实验分到总时限 ÷ 实验数。种子同正式运行（同一份研究共用一个）。 + */ +export function diagnosisRun(task: Task, imageDigest: string, seed: string, experimentIds: string[]): RunDefinition { + return implementation(task).diagnosisRun(task, imageDigest, seed, experimentIds); +} + +/** + * 诊断判分第一步：每个交出答案的实验的答案复制进 judgeDir 下以实验编号命名的子目录， + * 交回校验器的运行定义（一个容器逐个实验判分）；没有一个实验交出答案时直接交回判分结果。 + */ +export function prepareDiagnosisJudging( + task: Task, + runDir: string, + judgeDir: string, + imageDigest: string, + process: ProcessResult, + experiments: DiagnosisExperiment[], +): DiagnosisJudgingStep { + return implementation(task).prepareDiagnosisJudging(task, runDir, judgeDir, imageDigest, process, experiments); +} + +/** + * 诊断判分第二步：按实验分开交回分数、合法化、登记的指标、用时、结局。validator 是第一步交回过运行定义时跑出的结果， + * 第一步直接判完时传 null。资源用量记在整次运行的 ProcessResult 里，不在这里。 + * 校验器自己没正常跑完、判定文件认不出、修后分数高于原分数时报错停下，同正式运行的 judge。 + */ +export function judgeDiagnosis( + task: Task, + runDir: string, + judgeDir: string, + process: ProcessResult, + validator: ProcessResult | null, + experiments: DiagnosisExperiment[], +): DiagnosisJudged { + return implementation(task).judgeDiagnosis(task, runDir, judgeDir, process, validator, experiments); +} diff --git a/src/tasks/types.ts b/src/tasks/types.ts index 1645ecf..50d22bf 100644 --- a/src/tasks/types.ts +++ b/src/tasks/types.ts @@ -1,4 +1,4 @@ -import type { HypothesisMetric, JudgedResult, MetricDefinition, ProcessResult, RunDefinition } from "../vocab/index.ts"; +import type { HypothesisMetric, JudgedResult, Json, Measured, MetricDefinition, ProcessResult, RunDefinition, Verdict } from "../vocab/index.ts"; import type { TaskFacts } from "./facts.ts"; /** 读进来的一个任务。 */ @@ -22,6 +22,44 @@ export type LightCheckOutcome = { passed: true } | { passed: false; reasons: str */ export type JudgingStep = { status: "judged"; result: JudgedResult } | { status: "validate"; definition: RunDefinition }; +/** 一个要跑的诊断实验：编号，以及它自己登记的假设专用指标（冻结后的全名；诊断实验只能登记"缺了只记缺失"的指标）。 */ +export interface DiagnosisExperiment { + id: string; + metrics: HypothesisMetric[]; +} + +/** + * 一个诊断实验怎么结束：finished 跑完交出了答案；crashed 实验自己崩了；timeout 超过分到的时间被杀； + * notRun 整次运行先被总时限或从外面结束，没轮到它。 + */ +export type DiagnosisEnding = "finished" | "crashed" | "timeout" | "notRun"; + +/** 一个诊断实验单独判分的结果。资源用量（CPU、峰值内存）记在整次运行的结果里，这里只有它自己的用时。 */ +export interface DiagnosisExperimentJudged { + id: string; + ending: DiagnosisEnding; + /** 这个实验实际用掉的墙钟秒数；notRun 时测不到。 */ + wallSeconds: Measured; + verdict: Verdict; + verdictReason: string; + /** 合法化之后的最终指标分数；不算数时测不到并附原因。 */ + score: Measured; + /** 校验器给的辅助指标，加上这个实验登记的假设专用指标，键是指标编号。 */ + metrics: Record; + /** 同正式运行的 extra：填圆是合法化的明细。 */ + extra: Json; +} + +/** 一次诊断运行的判分结果：每个实验一项，顺序同运行定义里的实验顺序。 */ +export interface DiagnosisJudged { + experiments: DiagnosisExperimentJudged[]; +} + +/** 诊断判分第一步：不用跑校验器就已判定（没有一个实验交出答案），或交回校验器的运行定义。 */ +export type DiagnosisJudgingStep = + | { status: "judged"; result: DiagnosisJudged } + | { status: "validate"; definition: RunDefinition }; + /** 一种任务的实现。对外函数按 task.facts.kind 从对照表里取出它来调。参数的含义见 task.ts 里同名的对外函数。 */ export interface TaskImplementation { /** 最终指标（编号和方向取自 task.json）与常规辅助指标的定义。 */ @@ -46,4 +84,25 @@ export interface TaskImplementation { validator: ProcessResult, hypothesisMetrics: HypothesisMetric[], ): JudgedResult; + /** 诊断运行（方法.md 4.3"诊断"一段）：一次运行跑几个实验。上面七个函数的行为与它无关。 */ + diagnosisUnversionedNames(facts: TaskFacts): string[]; + placeBaselineArtifact(task: Task, baselineRunDir: string, dir: string): void; + prepareDiagnosisRun(task: Task, programDir: string, baselineRunDir: string, runDir: string, experiments: DiagnosisExperiment[]): void; + diagnosisRun(task: Task, imageDigest: string, seed: string, experimentIds: string[]): RunDefinition; + prepareDiagnosisJudging( + task: Task, + runDir: string, + judgeDir: string, + imageDigest: string, + process: ProcessResult, + experiments: DiagnosisExperiment[], + ): DiagnosisJudgingStep; + judgeDiagnosis( + task: Task, + runDir: string, + judgeDir: string, + process: ProcessResult, + validator: ProcessResult | null, + experiments: DiagnosisExperiment[], + ): DiagnosisJudged; } diff --git a/src/vocab/checklist.ts b/src/vocab/checklist.ts index 9bf3a87..ce90410 100644 --- a/src/vocab/checklist.ts +++ b/src/vocab/checklist.ts @@ -22,6 +22,9 @@ type FactorRules = { /** 实现、修复、核验共用：冻结协议的实现条款（c1…）、因子（p1…）、保障短名。 */ type FrozenItems = { candidate: string; clauses: string[]; factors: string[]; safeguards: string[] }; +/** 诊断实现、修复、核验共用：本次诊断选中的实验编号（d1…）。 */ +type DiagnosisItems = { experiments: string[] }; + /** 入口个数的范围（方法.md 4.2：2–3 个）。 */ type EntryCount = { min: 2; max: 3 }; @@ -30,6 +33,16 @@ export type Checklist = | { kind: "discussion"; entries: EntryCount } /** minChoices = min(N, 入口总数)（方法.md 4.2 认领）。 */ | { kind: "claim"; entries: string[]; minChoices: number } + /** + * 诊断解释最多 M 条、诊断实验最多 A 个（方法.md 4.3 诊 1）；takenMetricIds 是新登记的指标短名不能用的编号 + * （最终指标、常规辅助指标和答案路径的两个保留键）。 + */ + | { kind: "diagnosisProposal"; maxExplanations: number; maxExperiments: number; takenMetricIds: string[] } + /** 对自己每条解释（explanations）× 每个合并后的实验（experiments）各给三个数（方法.md 4.3 诊 2）。 */ + | { kind: "diagnosisPrediction"; explanations: string[]; experiments: string[] } + | ({ kind: "diagnosisImplementation" } & DiagnosisItems) + | ({ kind: "diagnosisRepair" } & DiagnosisItems) + | ({ kind: "diagnosisVerification" } & DiagnosisItems) | ({ kind: "proposal"; maxProposals: number } & FactorRules) | { kind: "firstPrediction"; candidate: string } | { kind: "secondPrediction"; candidate: string } diff --git a/src/vocab/roster.ts b/src/vocab/roster.ts index d44ac94..3364a4e 100644 --- a/src/vocab/roster.ts +++ b/src/vocab/roster.ts @@ -10,6 +10,10 @@ export const STAGES = [ "entryDraft", // 0.1 闭门写入口 "discussion", // 0.2 讨论，可配置为 1 或 2 次，每次都是这个阶段 "claim", // 0.3 认领 + "diagnosisProposal", // 诊 1 诊断解释与诊断实验 + "diagnosisPrediction", // 诊 2 诊断预测 + "diagnosisImplementation", // 诊 4 诊断实现，以及核验不通过后的修复 + "diagnosisVerification", // 诊 4 诊断核验 "proposal", // ② 提案 "firstPrediction", // ④ 第一次预测 "critique", // ⑤ 挑刺 @@ -33,6 +37,11 @@ export const SUBMISSION_KINDS = { entry: { stage: "entryDraft", role: "researcher" }, discussion: { stage: "discussion", role: "researcher" }, claim: { stage: "claim", role: "researcher" }, + diagnosisProposal: { stage: "diagnosisProposal", role: "researcher" }, + diagnosisPrediction: { stage: "diagnosisPrediction", role: "researcher" }, + diagnosisImplementation: { stage: "diagnosisImplementation", role: "experimenter" }, + diagnosisRepair: { stage: "diagnosisImplementation", role: "experimenter" }, + diagnosisVerification: { stage: "diagnosisVerification", role: "verifier" }, proposal: { stage: "proposal", role: "researcher" }, firstPrediction: { stage: "firstPrediction", role: "researcher" }, critique: { stage: "critique", role: "researcher" }, diff --git a/src/vocab/submissions.ts b/src/vocab/submissions.ts index 6a41ea1..3449294 100644 --- a/src/vocab/submissions.ts +++ b/src/vocab/submissions.ts @@ -503,6 +503,146 @@ export const VerificationSubmission = Type.Object( }, ); +// --------------------------------------------------------------------------- +// 诊断：解释与实验、预测、实现、核验（方法.md 4.3 诊 1–诊 4、第五部分"诊断"三行） +// --------------------------------------------------------------------------- + +const DiagnosisExplanation = Type.Object( + { + body: Type.String({ + ...TEXT, + description: + "Markdown：为什么卡在这里，剩下的空间在哪，什么样的实测会推翻它（写不出推翻它的实测，说明这条解释还不能被检验，先改具体再交）。" + + "诊断问的是剩下的空间在哪，不是上一步的功劳怎么分。没有具体判断就直说，写明查了什么、为什么拿不准", + }), + }, + CLOSED, +); + +const DiagnosisExperiment = Type.Object( + { + name: Type.String({ + ...TEXT, + description: "名称：离开这份清单单独看也读得懂", + }), + spot: Type.String({ + ...TEXT, + description: "动哪一处：在当前基线的哪一处代码或哪一步上动手，只写一处（文件、函数或步骤名），其余照基线保留；想测两处就拆成两个实验", + }), + body: Type.String({ + ...TEXT, + description: + "Markdown：在这一处怎么动（换成明显更强的版本、换成另一种做法、拿掉，或放宽某个限制）、" + + "做完交出什么能判分的答案、几种可能的结果各排除哪几条解释（有分不开的也写明）。不写自己估的分数，分数在下一步交", + }), + metrics: Type.Array(HypothesisMetric, { + description: + "登记假设专用指标：这个实验的代码要额外记录的量，编号只写短名；没有就交空数组。" + + "诊断实验里的指标只能选 recordMissing（缺了只记缺失），选 invalidateRun 会被退回", + }), + }, + CLOSED, +); + +export const DiagnosisProposalSubmission = Type.Object( + { + explanations: Type.Array(DiagnosisExplanation, { + minItems: 1, + description: "诊断解释清单，编号由程序分配；条数上限见材料。真有第二个看法再写第二条，各条之间要真的不同；没有具体判断，就写一条直说没有的", + }), + experiments: Type.Array(DiagnosisExperiment, { + minItems: 1, + description: + "诊断实验清单，个数上限见材料。好的实验是不同解释对它的预测差得远：大家都预测涨不了多少的实验，测了也不改变下一步。" + + "每个实验只动一处，用时要放得进分到的那份时间；想不出能把解释分开的实验，就交最能暴露你的解释错在哪的那个", + }), + }, + { + ...CLOSED, + description: "对当前基线的诊断：几条解释，和能把它们分开的几个实验。实验正文里不写自己估的分数", + }, +); + +export const DiagnosisPredictionSubmission = Type.Object( + { + predictions: Type.Array( + Type.Object( + { + explanation: Type.String({ ...Id, description: "你自己的诊断解释编号，照材料里列的写" }), + experiment: Type.String({ ...Id, description: "合并后的诊断实验编号，照材料里列的写，别人提的也要给" }), + range: Range, + }, + CLOSED, + ), + { + description: + "你自己的每条诊断解释对每个合并后的实验各给三个数：按这条解释，做了这个实验，分数有一成、一半、九成可能低于多少。" + + "不多不少，每一对只写一次；按这条解释本身写，不往折中的范围靠；解释和这个实验没关系就按基线附近写并放宽；拿不准就把 q1 和 q9 拉开", + }, + ), + body: Type.String({ + ...TEXT, + description: "理由，Markdown：这些数是怎么来的，不同解释给的数差在哪里；几条解释给的数差不多，写明它们在这些实验上分不开", + }), + }, + CLOSED, +); + +/** 诊断实现和修复用同一个定义。代码本身由程序从实验员的工作目录冻结，不作为字段交。 */ +export const DiagnosisImplementationSubmission = Type.Object( + { + locations: Type.Array( + Type.Object( + { + experiment: Type.String({ ...Id, description: "诊断实验编号，照材料里列的写" }), + locations: LOCATIONS, + }, + CLOSED, + ), + { + description: "位置表：选中的每个实验一项，列出本次代码中这个实验实际所在的全部位置，行号以本次提交代码为准", + }, + ), + concerns: CONCERNS, + body: Type.String({ + ...TEXT, + description: "实现说明，Markdown：位置表之外需要说明的事；实验原文没规定规模时，写你选了什么、为什么放得进分到的时间。修复时逐条回应核验意见：改在哪里，或以实验原文和代码位置说明为什么不改。没有要补充的就写\"无\"", + }), + }, + CLOSED, +); + +export const DiagnosisVerificationSubmission = Type.Object( + { + experiments: Type.Array( + Type.Object( + { + experiment: Type.String({ ...Id, description: "诊断实验编号，照材料里列的写" }), + followed: Type.Boolean({ + description: "照实验写的做法实现了写 true；没找到、做得不一样或材料不足以核清，写 false", + }), + nothingElse: Type.Boolean({ + description: "没有动实验写的那一处以外的地方写 true；顺手改了别处（含顺手改进）写 false", + }), + where: WHERE, + }, + CLOSED, + ), + { + description: "选中的每个实验一项，逐个判断，一个实验不通过不影响别的实验", + }, + ), + body: Type.String({ + ...TEXT, + description: "说明，Markdown：逐项结果之外需要说明的事。没有要补充的就写\"无\"", + }), + }, + { + ...CLOSED, + description: "只交逐个实验的结果，通过与否由程序推出：任何一项写 false 的实验退回实验员修复，用完修复次数仍不通过的实验从这次诊断里拿掉", + }, +); + // --------------------------------------------------------------------------- // 解释、消融、收尾（方法.md 4.3 ⑭⑯⑱） // --------------------------------------------------------------------------- @@ -611,6 +751,11 @@ export const SUBMISSION_SCHEMAS = { entry: EntrySubmission, discussion: DiscussionSubmission, claim: ClaimSubmission, + diagnosisProposal: DiagnosisProposalSubmission, + diagnosisPrediction: DiagnosisPredictionSubmission, + diagnosisImplementation: DiagnosisImplementationSubmission, + diagnosisRepair: DiagnosisImplementationSubmission, + diagnosisVerification: DiagnosisVerificationSubmission, proposal: ProposalSubmission, firstPrediction: PredictionSubmission, critique: CritiqueSubmission, diff --git a/tasks/circle_packing/TASK.md b/tasks/circle_packing/TASK.md index ae3cee7..d287b2f 100644 --- a/tasks/circle_packing/TASK.md +++ b/tasks/circle_packing/TASK.md @@ -154,6 +154,34 @@ import 之前它还会删掉工作目录里已有的 `solution.json` 和 `metric **不要自己跑正式规模的求解。** 正式运行要先登记、再启动、再扣次数，由程序统一做； 正式规模的求解哪怕叫"调试"，也是正式运行，不能借轻量检查或自己的 shell 绕开预算。 +## 诊断实验怎么拿到原始产物 + +诊断实验（在当前基线上只动一处、再看分数）不走上面的 `run_packing()`。实验员把本轮选中的全部实验写进**一份诊断程序**， +它从当前基线的代码出发，入口仍是 `initial.py`，但要定义的是： + +```python +def run_experiment(experiment_id, baseline_dir): + ... + return centers, radii, sum_of_radii # 与 run_packing() 的返回值同形 +``` + +- `experiment_id` 是实验编号原文，函数里按它分派到各个实验；每个实验只动它写的那一处，其余照基线。 +- `baseline_dir` 是**产出当前基线的那次正式运行的原始产物**所在的目录，相对工作目录，现在就是 `baseline-artifact`。 + 里面有一个文件 `solution.json`：当时那次运行交出的原始解（`centers`、`radii`、`reported_sum`，合法化之前）， + 实验要接着算的答案就在里面。程序把它**只读**地放进你的工作目录（文件权限 0444），运行时每个实验读到的是同一份； + 不要改它，也不要把它当成自己代码的一部分：存代码版本时 `baseline-artifact` 这个名字被跳过。 +- 一次诊断运行是**一个容器、一次正式运行的名额**，总时限和核数同上（120 秒、1 核）。容器里的诊断 harness + （`harness-diagnosis.py`，宿主保管，由程序放进运行目录）按实验顺序一个一个跑，每个实验开一个单独的进程， + 分到**总时限 ÷ 实验数**的时间，到时被直接杀掉。numba 的首次编译、数据载入都算在这份时间里。 + 一个实验崩了或超时，只记这一个实验失败，后面的照常跑；整次运行被总时限杀掉时，没轮到的实验记"没轮到"。 +- harness 记下每个实验实际用了多久、有没有超时。每个实验的答案写在 `answers/<实验编号>/solution.json`， + 由宿主的校验器按上面"评测会收下什么"的次序**逐个单独判分**（包括只缩不移合法化）。 +- 实验登记的假设专用指标，写法同下面"指标怎么交给宿主"：在 `run_experiment` 返回之前往工作目录的 `metrics.json` 写， + 键照抄 `metrics-spec.json` 里**这个实验自己登记**的指标编号（该文件里是本轮全部实验登记的指标，别的实验的不用写）。 + 每个实验开始前 harness 会清掉工作目录里的 `metrics.json`，不会串到下一个实验。诊断实验登记的指标一律是"缺了只记缺失"， + `answer-path` 和 `safeguard-triggers` 两个保留键不用写。 +- 诊断实验的答案**不会被采纳为基线**，只用来看这个方向还剩多少空间。 + ## 指标怎么交给宿主（这一节最容易写错，先读完再动手） 工作目录里有两个指标文件，一进一出。**路径都相对工作目录**，也就是 `harness.py` diff --git a/tasks/circle_packing/harness-diagnosis.py b/tasks/circle_packing/harness-diagnosis.py new file mode 100644 index 0000000..d732274 --- /dev/null +++ b/tasks/circle_packing/harness-diagnosis.py @@ -0,0 +1,125 @@ +"""填圆任务的诊断运行 harness（宿主的固定工具，候选不得修改）。 + +一次诊断运行是一个容器、一次正式运行的名额，里面按顺序跑若干个诊断实验（docs/方法.md 4.3"诊断"一段）。 +每个实验各开一个子进程：实验崩了、超时了，只影响它自己，后面的实验照常跑。 +**它不判定解是否合法**：每个实验交出的答案，事后由独立校验器 ``validate.py`` 逐个重算。 +读候选程序、写原始解、整理 metrics.json 都用 ``harness.py`` 里现成的函数，两份 harness 在这些事上不分叉。 + +命令行（由宿主的运行定义给出，实验员不用自己跑）： + python -B harness-diagnosis.py --program <程序> --baseline <目录> --answers <目录> + --seconds-each <秒> --report <文件> --experiments <实验编号> [<实验编号> ...] + +每个实验： + - 调用候选程序里的 ``run_experiment(experiment_id, baseline_dir)``，返回 (centers, radii, reported_sum)， + 与 ``run_packing()`` 的返回值同形。``experiment_id`` 是实验编号原文，``baseline_dir`` 是只读的原始产物目录 + （相对工作目录），里面是产出当前基线的那次正式运行的 ``solution.json``。 + - 分到的时限是 ``--seconds-each``（总时限 ÷ 实验数），到时被直接杀掉。 + - 随机种子同正式运行：环境变量 INFORSI_SEED，每个实验的子进程各自从头设一次。 + - 答案写进 ``/<实验编号>/solution.json``。指标写法同 TASK.md"指标怎么交给宿主"： + 在 ``run_experiment`` 返回之前往工作目录里的 ``metrics.json`` 写，写完后 harness 把它整理好、 + 搬到 ``/<实验编号>/metrics.json``，下一个实验开始前工作目录里不留它。 + +每个实验结束后，harness 重写一次 ``--report``：每个实验一项 +``{"id", "status", "wall_seconds"}``，status 是 finished（子进程退出码 0）、crashed（非零退出）、 +timeout（超过分到的时限，被杀）。跑到哪写到哪：整个容器被总时限杀掉时，已经跑完的实验还留着记录， +没跑到的实验没有记录。 +""" + +import argparse +import hashlib +import importlib +import json +import os +import random +import signal +import subprocess +import sys +import time +from typing import Any, cast + +import harness + +SOLUTION_FILE = "solution.json" + + +def run_experiment_child(args) -> int: + """子进程：设种子、载入候选程序、跑一个实验、落盘答案和指标。""" + np = importlib.import_module("numpy") + + seed = int.from_bytes( + hashlib.sha256(os.environ["INFORSI_SEED"].encode()).digest()[:4], "big" + ) + random.seed(seed) + np.random.seed(seed) + + module = harness.load_program(args.program) + run_experiment = getattr(module, "run_experiment", None) + if not callable(run_experiment): + raise RuntimeError(f"{args.program} 没有定义可调用的 run_experiment(experiment_id, baseline_dir)") + centers, radii, reported_sum = cast(Any, run_experiment(args.id, args.baseline)) + solution = { + "centers": [[float(x), float(y)] for x, y in centers], + "radii": [float(r) for r in radii], + "reported_sum": float(reported_sum), + } + harness.write_json_atomically(args.output, solution) + harness.finalize_metrics() + return 0 + + +def run_experiment_with_limit(args, experiment_id: str) -> tuple[str, float]: + """在子进程里跑一个实验，交回结局和用时。""" + answer_dir = os.path.join(args.answers, experiment_id) + os.makedirs(answer_dir) + # 上一个实验或候选目录里遗留的指标文件，不能被当成这个实验的。 + if os.path.exists(harness.METRICS_FILE): + os.remove(harness.METRICS_FILE) + command = [ + sys.executable, "-B", os.path.abspath(__file__), "--child", + "--program", args.program, "--baseline", args.baseline, + "--id", experiment_id, "--output", os.path.join(answer_dir, SOLUTION_FILE), + ] + start = time.monotonic() + # 新开一个进程组：超时时连实验自己开的子进程一起杀掉。 + child = subprocess.Popen(command, start_new_session=True) + try: + status = "finished" if child.wait(timeout=args.seconds_each) == 0 else "crashed" + except subprocess.TimeoutExpired: + os.killpg(child.pid, signal.SIGKILL) + child.wait() + status = "timeout" + wall = time.monotonic() - start + if os.path.exists(harness.METRICS_FILE): + if status == "finished": + os.replace(harness.METRICS_FILE, os.path.join(answer_dir, harness.METRICS_FILE)) + else: + os.remove(harness.METRICS_FILE) + return status, wall + + +def main() -> int: + parser = argparse.ArgumentParser(description="按顺序运行若干个诊断实验，各自限时、互不影响。") + parser.add_argument("--program", required=True) + parser.add_argument("--baseline", required=True) + parser.add_argument("--child", action="store_true") + parser.add_argument("--id") + parser.add_argument("--output") + parser.add_argument("--answers") + parser.add_argument("--seconds-each", type=float) + parser.add_argument("--report") + parser.add_argument("--experiments", nargs="+") + args = parser.parse_args() + if args.child: + return run_experiment_child(args) + + report: list[dict] = [] + for experiment_id in args.experiments: + status, wall = run_experiment_with_limit(args, experiment_id) + print(f"[诊断实验 {experiment_id}] {status}，用时 {wall:.2f} 秒", file=sys.stderr, flush=True) + report.append({"id": experiment_id, "status": status, "wall_seconds": wall}) + harness.write_json_atomically(args.report, report) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/config-check.test.ts b/tests/config-check.test.ts index 1ae7573..534d2d9 100644 --- a/tests/config-check.test.ts +++ b/tests/config-check.test.ts @@ -177,6 +177,7 @@ test("写不出来的组合：朴素循环里写开关、规则或它用不到 critique: false, firstPrediction: false, roundZero: { enabled: false }, + diagnosis: { enabled: false }, ablation: { enabled: false }, batchRule: "uniformRandom", openReserve: 0.1, @@ -200,6 +201,35 @@ test("写不出来的组合：关掉的第零轮写讨论次数、关掉的消 assert.match(errorOf(ablation), /^\/flow\/ablation\/cellRule：不认识的字段$/m); }); +/** 诊断开着的完整流程配置。 */ +function diagnosisConfig(): ReturnType { + const config = fullConfig(); + (config.flow as Obj).diagnosis = { enabled: true, perResearcher: 2, maxExperiments: 4 }; + return config; +} + +test("诊断：开着的配置通过校验；逐个删字段、写 0、写小数、多写字段都报错；关着的不能写两个参数", () => { + assert.deepEqual(checkConfig(structuredClone(diagnosisConfig())), diagnosisConfig()); + + for (const key of ["enabled", "perResearcher", "maxExperiments"]) { + const config = diagnosisConfig(); + delete ((config.flow as Obj).diagnosis as Obj)[key]; + assert.match(errorOf(config), new RegExp(`^/flow/diagnosis/${key}：`, "m")); + } + const cases: Array<[Obj, RegExp]> = [ + [{ enabled: true, perResearcher: 0, maxExperiments: 4 }, /^\/flow\/diagnosis\/perResearcher：不能小于 1$/m], + [{ enabled: true, perResearcher: 2, maxExperiments: 2.5 }, /^\/flow\/diagnosis\/maxExperiments：应为整数$/m], + [{ enabled: true, perResearcher: 2, maxExperiments: 4, rule: "x" }, /^\/flow\/diagnosis\/rule：不认识的字段$/m], + [{ enabled: false, perResearcher: 2 }, /^\/flow\/diagnosis\/perResearcher：不认识的字段$/m], + [{ enabled: "yes" }, /^\/flow\/diagnosis\/enabled：缺少或取值不对，只能是 false、true 之一$/m], + ]; + for (const [diagnosis, expected] of cases) { + const config = fullConfig(); + (config.flow as Obj).diagnosis = diagnosis; + assert.match(errorOf(config), expected); + } +}); + test("写不出来的组合：规则名不在 rules 登记表里", () => { const batch = fullConfig(); (batch.flow as Obj).batchRule = "expected-best"; diff --git a/tests/config-helpers.ts b/tests/config-helpers.ts index 21eaf13..63fcfd5 100644 --- a/tests/config-helpers.ts +++ b/tests/config-helpers.ts @@ -29,6 +29,7 @@ export function fullConfig(): ResearchConfig { roundZero: { enabled: true, discussions: 2 }, firstPrediction: true, critique: true, + diagnosis: { enabled: false }, ablation: { enabled: true, cellRule: "valueMinusCost" }, batchRule: "bestExpected", openReserve: 0.1, diff --git a/tests/config-steps.test.ts b/tests/config-steps.test.ts index e3fbeca..22b3eee 100644 --- a/tests/config-steps.test.ts +++ b/tests/config-steps.test.ts @@ -28,7 +28,7 @@ function assertReasons(list: StepList): void { } test("步骤清单：完整流程全开", () => { - const list = stepList(fullConfig()); + const list = listOf((f) => (f.diagnosis = { enabled: true, perResearcher: 2, maxExperiments: 4 })); assert.equal(list.preset, "full"); assert.deepEqual(rows(list.study), [ ["freezeConfig", E, null], @@ -44,6 +44,13 @@ test("步骤清单：完整流程全开", () => { assert.deepEqual(rows(list.round), [ ["openRound", E, null], ["carryOverPool", E, null], + ["diagnosisProposal", E, null], + ["diagnosisDedupe", E, null], + ["diagnosisPrediction", E, null], + ["selectDiagnosis", E, { name: "valueMinusCost", version: 1 }], + ["diagnosisImplementation", E, null], + ["diagnosisRun", E, null], + ["diagnosisSettle", E, null], ["proposal", E, null], ["dedupe", E, null], ["firstPrediction", E, null], @@ -95,6 +102,13 @@ test("步骤清单：只关第一次预测", () => { assert.deepEqual(rows(list.round), [ ["openRound", E, null], ["carryOverPool", E, null], + ["diagnosisProposal", S, null], + ["diagnosisDedupe", S, null], + ["diagnosisPrediction", S, null], + ["selectDiagnosis", S, null], + ["diagnosisImplementation", S, null], + ["diagnosisRun", S, null], + ["diagnosisSettle", S, null], ["proposal", E, null], ["dedupe", E, null], ["firstPrediction", S, null], @@ -128,6 +142,13 @@ test("步骤清单：只关挑刺与回应，协议照样冻结", () => { assert.deepEqual(rows(list.round), [ ["openRound", E, null], ["carryOverPool", E, null], + ["diagnosisProposal", S, null], + ["diagnosisDedupe", S, null], + ["diagnosisPrediction", S, null], + ["selectDiagnosis", S, null], + ["diagnosisImplementation", S, null], + ["diagnosisRun", S, null], + ["diagnosisSettle", S, null], ["proposal", E, null], ["dedupe", E, null], ["firstPrediction", E, null], @@ -154,13 +175,20 @@ test("步骤清单：只关挑刺与回应，协议照样冻结", () => { ]); }); -test("步骤清单：只关消融，解释照样写，选格没有规则", () => { +test("步骤清单：只关消融，解释也跳过，选格没有规则", () => { const list = listOf((f) => (f.ablation = { enabled: false })); assertReasons(list); assert.deepEqual(rows(list.study), rows(stepList(fullConfig()).study)); assert.deepEqual(rows(list.round), [ ["openRound", E, null], ["carryOverPool", E, null], + ["diagnosisProposal", S, null], + ["diagnosisDedupe", S, null], + ["diagnosisPrediction", S, null], + ["selectDiagnosis", S, null], + ["diagnosisImplementation", S, null], + ["diagnosisRun", S, null], + ["diagnosisSettle", S, null], ["proposal", E, null], ["dedupe", E, null], ["firstPrediction", E, null], @@ -177,7 +205,7 @@ test("步骤清单：只关消融，解释照样写，选格没有规则", () => ["informationAccount", E, null], ["adopt", E, null], ["listCells", S, null], - ["explanation", E, null], + ["explanation", S, null], ["selectCells", S, null], ["ablationImplementation", S, null], ["ablationVerification", S, null], @@ -205,6 +233,13 @@ test("步骤清单：朴素循环", () => { assert.deepEqual(rows(list.round), [ ["openRound", E, null], ["carryOverPool", S, null], // 候选不留池 + ["diagnosisProposal", S, null], // 不诊断 + ["diagnosisDedupe", S, null], // 不诊断 + ["diagnosisPrediction", S, null], // 不诊断 + ["selectDiagnosis", S, null], // 不诊断 + ["diagnosisImplementation", S, null], // 不诊断 + ["diagnosisRun", S, null], // 不诊断 + ["diagnosisSettle", S, null], // 不诊断 ["proposal", E, null], ["dedupe", E, null], // 判重照常 ["firstPrediction", S, null], // 不预测 @@ -264,3 +299,30 @@ test("步骤清单：换规则后清单里的规则跟着换", () => { assert.deepEqual(rule("selectBatch"), { name: "uniformRandom", version: 1 }); assert.deepEqual(rule("selectCells"), { name: "randomSameCount", version: 1 }); }); + +test("步骤清单：诊断开、消融关（现行配置）：诊断七步执行，消融七步（含解释）跳过", () => { + const list = listOf((f) => { + f.diagnosis = { enabled: true, perResearcher: 2, maxExperiments: 4 }; + f.ablation = { enabled: false }; + }); + assertReasons(list); + const status = (id: string) => list.round.find((s) => s.id === id)!.status; + for (const id of [ + "diagnosisProposal", "diagnosisDedupe", "diagnosisPrediction", "selectDiagnosis", + "diagnosisImplementation", "diagnosisRun", "diagnosisSettle", + ]) { + assert.equal(status(id), "execute", id); + } + for (const id of [ + "listCells", "explanation", "selectCells", "ablationImplementation", "ablationVerification", "ablationRun", "settle", + ]) { + assert.equal(status(id), "skip", id); + } + // 诊断七步排在留池之后、提案之前（方法.md 4.3"诊断"一段）。 + const ids = list.round.map((s) => s.id); + assert.deepEqual(ids.slice(ids.indexOf("carryOverPool"), ids.indexOf("proposal") + 1), [ + "carryOverPool", "diagnosisProposal", "diagnosisDedupe", "diagnosisPrediction", "selectDiagnosis", + "diagnosisImplementation", "diagnosisRun", "diagnosisSettle", "proposal", + ]); + assert.deepEqual(list.round.find((s) => s.id === "selectDiagnosis")!.rule, { name: "valueMinusCost", version: 1 }); +}); diff --git a/tests/fixtures/rules/diagnosis.json b/tests/fixtures/rules/diagnosis.json new file mode 100644 index 0000000..f64b11a --- /dev/null +++ b/tests/fixtures/rules/diagnosis.json @@ -0,0 +1,1937 @@ +{ + "generator": "gen_diagnosis.py", + "tolerance": { + "exact": "有理数精确算出的量（成分权重、事前期望收益、ĝ₀、全做的 VOI、R̂、m′、成本、价值、净值、收益、更新后的权重）：绝对误差 1e-12", + "exactK1": "只做一个实验的 VOI 是逐段闭式积分，实现用部分格网格近似：|差| ≤ 1e-9 + 2e-4·|精确值|", + "bits": "世界比特、惊讶、分歧量是浮点：绝对误差 1e-9；合并范围是分位点：绝对误差 1e-9", + "discrete": "是否诊断、选中的实验、确认与否定、有没有指出方向、实验的结局：必须完全相同" + }, + "scenarios": [ + { + "name": "opposite_lambda0", + "section": "6.12", + "input": { + "b": 0.0, + "direction": "maximize", + "lambda": 0.0, + "experiments": [ + "d1" + ], + "explanations": [ + { + "id": "e1", + "author": "researcher-1", + "ranges": { + "d1": { + "q1": 0.15, + "q5": 0.2, + "q9": 0.25 + } + } + }, + { + "id": "e2", + "author": "researcher-2", + "ranges": { + "d1": { + "q1": -0.25, + "q5": -0.2, + "q9": -0.15 + } + } + } + ], + "balance": 7, + "batchCap": 2, + "maxExperiments": 4, + "u": 0.5, + "decay": 0.5 + }, + "expected": { + "priorGains": { + "d1": 0.0 + }, + "g0": 0.0, + "weights": [ + 0.5, + 0.5, + 0.0 + ], + "combos": [ + { + "experiments": [ + "d1" + ], + "voi": 0.1, + "rHat": 3, + "mPrime": 1.75, + "value": 0.0875, + "cost": 0.0, + "net": 0.0875 + } + ], + "diagnose": true, + "chosen": [ + "d1" + ], + "chosenNet": 0.0875 + }, + "note": "λ = 0，两个成分的范围不相交：做了就知道是哪一条。手算：做了以后 E[max(0, g)] = 0.5 × 0.2 = 0.1，不做时 max(0, E g) = max(0, 0.5×0.2 + 0.5×(−0.2)) = 0，VOI = 0.1。" + }, + { + "name": "opposite_lambda01", + "section": "6.12", + "input": { + "b": 0.0, + "direction": "maximize", + "lambda": 0.1, + "experiments": [ + "d1" + ], + "explanations": [ + { + "id": "e1", + "author": "researcher-1", + "ranges": { + "d1": { + "q1": 0.15, + "q5": 0.2, + "q9": 0.25 + } + } + }, + { + "id": "e2", + "author": "researcher-2", + "ranges": { + "d1": { + "q1": -0.25, + "q5": -0.2, + "q9": -0.15 + } + } + } + ], + "balance": 7, + "batchCap": 2, + "maxExperiments": 4, + "u": 0.5, + "decay": 0.5 + }, + "expected": { + "priorGains": { + "d1": 0.0 + }, + "g0": 0.0, + "weights": [ + 0.45, + 0.45, + 0.1 + ], + "combos": [ + { + "experiments": [ + "d1" + ], + "voi": 0.105, + "rHat": 3, + "mPrime": 1.75, + "value": 0.091875, + "cost": 0.0, + "net": 0.091875 + } + ], + "diagnose": true, + "chosen": [ + "d1" + ], + "chosenNet": 0.091875 + }, + "note": "同上，λ = 0.1，开放预留铺在两端之外。" + }, + { + "name": "same_lambda0", + "section": "6.12", + "input": { + "b": 0.0, + "direction": "maximize", + "lambda": 0.0, + "experiments": [ + "d1" + ], + "explanations": [ + { + "id": "e1", + "author": "researcher-1", + "ranges": { + "d1": { + "q1": 0.1, + "q5": 0.2, + "q9": 0.3 + } + } + }, + { + "id": "e2", + "author": "researcher-2", + "ranges": { + "d1": { + "q1": 0.1, + "q5": 0.2, + "q9": 0.3 + } + } + } + ], + "balance": 7, + "batchCap": 2, + "maxExperiments": 4, + "u": 0.5, + "decay": 0.5 + }, + "expected": { + "priorGains": { + "d1": 0.2 + }, + "g0": 0.2, + "weights": [ + 0.5, + 0.5, + 0.0 + ], + "combos": [ + { + "experiments": [ + "d1" + ], + "voi": 0.0, + "rHat": 3, + "mPrime": 1.75, + "value": 0.0, + "cost": 0.03333333333333333, + "net": -0.03333333333333333 + } + ], + "diagnose": false, + "chosen": [], + "chosenNet": 0.0 + }, + "note": "λ = 0，两条解释一样，做不做都不改变看法：VOI = 0（手算：g 的取值范围 0 到 0.4 全在 0 以上，E[max(0, g)] = E g = 0.2）。净值 = −成本 < 0，不诊断。" + }, + { + "name": "same_lambda01", + "section": "6.12", + "input": { + "b": 0.0, + "direction": "maximize", + "lambda": 0.1, + "experiments": [ + "d1" + ], + "explanations": [ + { + "id": "e1", + "author": "researcher-1", + "ranges": { + "d1": { + "q1": 0.1, + "q5": 0.2, + "q9": 0.3 + } + } + }, + { + "id": "e2", + "author": "researcher-2", + "ranges": { + "d1": { + "q1": 0.1, + "q5": 0.2, + "q9": 0.3 + } + } + } + ], + "balance": 7, + "batchCap": 2, + "maxExperiments": 4, + "u": 0.5, + "decay": 0.5 + }, + "expected": { + "priorGains": { + "d1": 0.2 + }, + "g0": 0.2, + "weights": [ + 0.45, + 0.45, + 0.1 + ], + "combos": [ + { + "experiments": [ + "d1" + ], + "voi": 0.0025, + "rHat": 3, + "mPrime": 1.75, + "value": 0.0021874999999999998, + "cost": 0.03333333333333333, + "net": -0.031145833333333334 + } + ], + "diagnose": false, + "chosen": [], + "chosenNet": 0.0 + }, + "note": "λ = 0.1：解释们的范围都在 0 以上，只有开放预留（铺在 −0.2 到 0.6）能给出负的结果，所以 VOI 只剩预留带来的一小部分。" + }, + { + "name": "multi_all", + "section": "6.12", + "input": { + "b": 0.0, + "direction": "maximize", + "lambda": 0.1, + "experiments": [ + "d1", + "d2", + "d3" + ], + "explanations": [ + { + "id": "e1", + "author": "researcher-1", + "ranges": { + "d1": { + "q1": 0.02, + "q5": 0.05, + "q9": 0.08 + }, + "d2": { + "q1": -0.05, + "q5": 0.0, + "q9": 0.05 + }, + "d3": { + "q1": 0.0, + "q5": 0.1, + "q9": 0.2 + } + } + }, + { + "id": "e2", + "author": "researcher-2", + "ranges": { + "d1": { + "q1": 0.0, + "q5": 0.03, + "q9": 0.06 + }, + "d2": { + "q1": 0.1, + "q5": 0.2, + "q9": 0.3 + }, + "d3": { + "q1": -0.1, + "q5": -0.05, + "q9": 0.0 + } + } + }, + { + "id": "e3", + "author": "researcher-3", + "ranges": { + "d1": { + "q1": -0.02, + "q5": 0.04, + "q9": 0.1 + }, + "d2": { + "q1": -0.2, + "q5": -0.1, + "q9": 0.0 + }, + "d3": { + "q1": 0.05, + "q5": 0.1, + "q9": 0.15 + } + } + } + ], + "balance": 9, + "batchCap": 2, + "maxExperiments": 4, + "u": 0.5, + "decay": 0.5 + }, + "expected": { + "priorGains": { + "d1": 0.04, + "d2": 0.034999999999999996, + "d3": 0.052500000000000005 + }, + "g0": 0.052500000000000005, + "weights": [ + 0.3, + 0.3, + 0.3, + 0.1 + ], + "combos": [ + { + "experiments": [ + "d1", + "d2", + "d3" + ], + "voi": 0.10650259407848324, + "rHat": 4, + "mPrime": 1.875, + "value": 0.09984618194857804, + "cost": 0.00875, + "net": 0.09109618194857805 + } + ], + "diagnose": true, + "chosen": [ + "d1", + "d2", + "d3" + ], + "chosenNet": 0.09109618194857805 + }, + "note": "三个实验不超过 D = 4，只有全做一个组合。R̂ = ⌈8 ÷ 2⌉ = 4，m′ = 1.875，成本 = 2ĝ₀ × (3/4 − 2/3)。" + }, + { + "name": "over_cap_d1", + "section": "6.12", + "input": { + "b": 0.0, + "direction": "maximize", + "lambda": 0.1, + "experiments": [ + "d1", + "d2", + "d3" + ], + "explanations": [ + { + "id": "e1", + "author": "researcher-1", + "ranges": { + "d1": { + "q1": 0.02, + "q5": 0.05, + "q9": 0.08 + }, + "d2": { + "q1": -0.05, + "q5": 0.0, + "q9": 0.05 + }, + "d3": { + "q1": 0.0, + "q5": 0.1, + "q9": 0.2 + } + } + }, + { + "id": "e2", + "author": "researcher-2", + "ranges": { + "d1": { + "q1": 0.0, + "q5": 0.03, + "q9": 0.06 + }, + "d2": { + "q1": 0.1, + "q5": 0.2, + "q9": 0.3 + }, + "d3": { + "q1": -0.1, + "q5": -0.05, + "q9": 0.0 + } + } + }, + { + "id": "e3", + "author": "researcher-3", + "ranges": { + "d1": { + "q1": -0.02, + "q5": 0.04, + "q9": 0.1 + }, + "d2": { + "q1": -0.2, + "q5": -0.1, + "q9": 0.0 + }, + "d3": { + "q1": 0.05, + "q5": 0.1, + "q9": 0.15 + } + } + } + ], + "balance": 9, + "batchCap": 2, + "maxExperiments": 1, + "u": 0.5, + "decay": 0.5 + }, + "expected": { + "priorGains": { + "d1": 0.04, + "d2": 0.034999999999999996, + "d3": 0.052500000000000005 + }, + "g0": 0.052500000000000005, + "weights": [ + 0.3, + 0.3, + 0.3, + 0.1 + ], + "combos": [ + { + "experiments": [ + "d1" + ], + "voi": 0.025331646896146795, + "rHat": 4, + "mPrime": 1.875, + "value": 0.02374841896513762, + "cost": 0.00875, + "net": 0.01499841896513762 + }, + { + "experiments": [ + "d2" + ], + "voi": 0.08911080457463884, + "rHat": 4, + "mPrime": 1.875, + "value": 0.08354137928872392, + "cost": 0.00875, + "net": 0.07479137928872391 + }, + { + "experiments": [ + "d3" + ], + "voi": 0.08340189036381514, + "rHat": 4, + "mPrime": 1.875, + "value": 0.0781892722160767, + "cost": 0.00875, + "net": 0.0694392722160767 + } + ], + "diagnose": true, + "chosen": [ + "d2" + ], + "chosenNet": 0.07479137928872391 + }, + "note": "三个实验超过 D = 1：只有三个单实验组合，没有空组合；精确值用逐段闭式积分。" + }, + { + "name": "over_cap_d2", + "section": "6.12", + "input": { + "b": 0.0, + "direction": "maximize", + "lambda": 0.1, + "experiments": [ + "d1", + "d2", + "d3" + ], + "explanations": [ + { + "id": "e1", + "author": "researcher-1", + "ranges": { + "d1": { + "q1": 0.02, + "q5": 0.05, + "q9": 0.08 + }, + "d2": { + "q1": -0.05, + "q5": 0.0, + "q9": 0.05 + }, + "d3": { + "q1": 0.0, + "q5": 0.1, + "q9": 0.2 + } + } + }, + { + "id": "e2", + "author": "researcher-2", + "ranges": { + "d1": { + "q1": 0.0, + "q5": 0.03, + "q9": 0.06 + }, + "d2": { + "q1": 0.1, + "q5": 0.2, + "q9": 0.3 + }, + "d3": { + "q1": -0.1, + "q5": -0.05, + "q9": 0.0 + } + } + }, + { + "id": "e3", + "author": "researcher-3", + "ranges": { + "d1": { + "q1": -0.02, + "q5": 0.04, + "q9": 0.1 + }, + "d2": { + "q1": -0.2, + "q5": -0.1, + "q9": 0.0 + }, + "d3": { + "q1": 0.05, + "q5": 0.1, + "q9": 0.15 + } + } + } + ], + "balance": 9, + "batchCap": 2, + "maxExperiments": 2, + "u": 0.5, + "decay": 0.5 + }, + "expected": { + "priorGains": { + "d1": 0.04, + "d2": 0.034999999999999996, + "d3": 0.052500000000000005 + }, + "g0": 0.052500000000000005, + "weights": [ + 0.3, + 0.3, + 0.3, + 0.1 + ], + "combos": [ + { + "experiments": [ + "d1", + "d2" + ], + "voi": null + }, + { + "experiments": [ + "d1", + "d3" + ], + "voi": null + }, + { + "experiments": [ + "d2", + "d3" + ], + "voi": null + } + ] + }, + "note": "三个实验超过 D = 2：只有三个两实验组合，没有单实验组合；只检查组合列表（网格近似没有精确值）。" + }, + { + "name": "balance_one", + "section": "6.12", + "input": { + "b": 0.0, + "direction": "maximize", + "lambda": 0.1, + "experiments": [ + "d1", + "d2", + "d3" + ], + "explanations": [ + { + "id": "e1", + "author": "researcher-1", + "ranges": { + "d1": { + "q1": 0.02, + "q5": 0.05, + "q9": 0.08 + }, + "d2": { + "q1": -0.05, + "q5": 0.0, + "q9": 0.05 + }, + "d3": { + "q1": 0.0, + "q5": 0.1, + "q9": 0.2 + } + } + }, + { + "id": "e2", + "author": "researcher-2", + "ranges": { + "d1": { + "q1": 0.0, + "q5": 0.03, + "q9": 0.06 + }, + "d2": { + "q1": 0.1, + "q5": 0.2, + "q9": 0.3 + }, + "d3": { + "q1": -0.1, + "q5": -0.05, + "q9": 0.0 + } + } + }, + { + "id": "e3", + "author": "researcher-3", + "ranges": { + "d1": { + "q1": -0.02, + "q5": 0.04, + "q9": 0.1 + }, + "d2": { + "q1": -0.2, + "q5": -0.1, + "q9": 0.0 + }, + "d3": { + "q1": 0.05, + "q5": 0.1, + "q9": 0.15 + } + } + } + ], + "balance": 1, + "batchCap": 2, + "maxExperiments": 4, + "u": 0.5, + "decay": 0.5 + }, + "expected": { + "priorGains": { + "d1": 0.04, + "d2": 0.034999999999999996, + "d3": 0.052500000000000005 + }, + "g0": 0.052500000000000005, + "weights": [ + 0.3, + 0.3, + 0.3, + 0.1 + ], + "combos": [], + "diagnose": false, + "chosen": [], + "chosenNet": 0.0 + }, + "note": "余额 1：没有可选组合，不诊断。" + }, + { + "name": "tie_by_id", + "section": "6.12", + "input": { + "b": 0.0, + "direction": "maximize", + "lambda": 0.1, + "experiments": [ + "d10", + "d2" + ], + "explanations": [ + { + "id": "e1", + "author": "researcher-1", + "ranges": { + "d2": { + "q1": -0.2, + "q5": 0.0, + "q9": 0.2 + }, + "d10": { + "q1": -0.2, + "q5": 0.0, + "q9": 0.2 + } + } + }, + { + "id": "e2", + "author": "researcher-2", + "ranges": { + "d2": { + "q1": 0.1, + "q5": 0.3, + "q9": 0.5 + }, + "d10": { + "q1": 0.1, + "q5": 0.3, + "q9": 0.5 + } + } + } + ], + "balance": 7, + "batchCap": 2, + "maxExperiments": 1, + "u": 0.5, + "decay": 0.5 + }, + "expected": { + "priorGains": { + "d2": 0.15, + "d10": 0.15 + }, + "g0": 0.15, + "weights": [ + 0.45, + 0.45, + 0.1 + ], + "combos": [ + { + "experiments": [ + "d2" + ], + "voi": 0.07180975286849074, + "rHat": 3, + "mPrime": 1.75, + "value": 0.0628335337599294, + "cost": 0.025, + "net": 0.03783353375992939 + }, + { + "experiments": [ + "d10" + ], + "voi": 0.07180975286849074, + "rHat": 3, + "mPrime": 1.75, + "value": 0.0628335337599294, + "cost": 0.025, + "net": 0.03783353375992939 + } + ], + "diagnose": true, + "chosen": [ + "d2" + ], + "chosenNet": 0.03783353375992939 + }, + "note": "两个实验的预测完全一样，净值相同（差不到 1e-12），取实验编号小的 d2。" + }, + { + "name": "settle_mixed", + "section": "6.12 结算；6.9", + "input": { + "b": 1.0, + "direction": "maximize", + "lambda": 0.1, + "experiments": [ + "d1", + "d2", + "d3" + ], + "explanations": [ + { + "id": "e1", + "author": "researcher-1", + "ranges": { + "d1": { + "q1": 1.02, + "q5": 1.05, + "q9": 1.08 + }, + "d2": { + "q1": 0.95, + "q5": 1.0, + "q9": 1.05 + }, + "d3": { + "q1": 1.0, + "q5": 1.1, + "q9": 1.2 + } + } + }, + { + "id": "e2", + "author": "researcher-2", + "ranges": { + "d1": { + "q1": 1.0, + "q5": 1.03, + "q9": 1.06 + }, + "d2": { + "q1": 1.1, + "q5": 1.2, + "q9": 1.3 + }, + "d3": { + "q1": 0.9, + "q5": 0.95, + "q9": 1.0 + } + } + }, + { + "id": "e3", + "author": "researcher-2", + "ranges": { + "d1": { + "q1": 0.98, + "q5": 1.04, + "q9": 1.1 + }, + "d2": { + "q1": 0.8, + "q5": 0.9, + "q9": 1.0 + }, + "d3": { + "q1": 1.05, + "q5": 1.1, + "q9": 1.15 + } + } + } + ], + "epsilon": 0.01, + "outcomes": { + "d1": { + "kind": "valid", + "score": 1.045 + }, + "d2": { + "kind": "insufficient" + }, + "d3": { + "kind": "untested" + } + } + }, + "expected": { + "experiments": { + "d1": { + "outcome": "valid", + "merged": { + "q1": 0.992, + "q5": 1.0428015564202335, + "q9": 1.0922077922077922 + }, + "gain": 0.04499999999999993, + "worldBits": 10.862272556463013, + "surprise": { + "status": "ok", + "bin": 450, + "p": 0.0010708333333332145, + "surprise": 9.867050331076888 + }, + "disagreement": 0.14778939669397495 + }, + "d2": { + "outcome": "insufficient", + "merged": { + "q1": 0.8466911764705882, + "q5": 1.0063258232235701, + "q9": 1.2533088235294119 + } + }, + "d3": { + "outcome": "untested", + "merged": { + "q1": 0.9198823529411765, + "q5": 1.068937125748503, + "q9": 1.192 + } + } + }, + "judgments": { + "e1": { + "d1": "confirmed" + }, + "e2": { + "d1": "confirmed" + }, + "e3": { + "d1": "confirmed" + } + }, + "direction": { + "weights": [ + 0.5603112840466926, + 0.2801556420233463, + 0.14007782101167315, + 0.019455252918287938 + ], + "weightsKept": false, + "gains": { + "d1": { + "gain": 0.04499999999999993, + "measured": true + }, + "d2": { + "gain": 0.042996108949416356, + "measured": false + }, + "d3": { + "gain": 0.05749027237354086, + "measured": false + } + }, + "experiment": "d3", + "gain": 0.05749027237354086, + "found": true + } + }, + "note": "d1 有效，d2 证据不足（不判、不记账），d3 没做。研究者 2 交了两条解释，各占他那份的一半。" + }, + { + "name": "settle_two_valid", + "section": "6.12 结算；6.9", + "input": { + "b": 1.0, + "direction": "maximize", + "lambda": 0.1, + "experiments": [ + "d1", + "d2", + "d3" + ], + "explanations": [ + { + "id": "e1", + "author": "researcher-1", + "ranges": { + "d1": { + "q1": 1.02, + "q5": 1.05, + "q9": 1.08 + }, + "d2": { + "q1": 0.95, + "q5": 1.0, + "q9": 1.05 + }, + "d3": { + "q1": 1.0, + "q5": 1.1, + "q9": 1.2 + } + } + }, + { + "id": "e2", + "author": "researcher-2", + "ranges": { + "d1": { + "q1": 1.0, + "q5": 1.03, + "q9": 1.06 + }, + "d2": { + "q1": 1.1, + "q5": 1.2, + "q9": 1.3 + }, + "d3": { + "q1": 0.9, + "q5": 0.95, + "q9": 1.0 + } + } + }, + { + "id": "e3", + "author": "researcher-2", + "ranges": { + "d1": { + "q1": 0.98, + "q5": 1.04, + "q9": 1.1 + }, + "d2": { + "q1": 0.8, + "q5": 0.9, + "q9": 1.0 + }, + "d3": { + "q1": 1.05, + "q5": 1.1, + "q9": 1.15 + } + } + } + ], + "epsilon": 0.01, + "outcomes": { + "d1": { + "kind": "valid", + "score": 1.07 + }, + "d2": { + "kind": "valid", + "score": 1.25 + }, + "d3": { + "kind": "untested" + } + } + }, + "expected": { + "experiments": { + "d1": { + "outcome": "valid", + "merged": { + "q1": 0.992, + "q5": 1.0428015564202335, + "q9": 1.0922077922077922 + }, + "gain": 0.07000000000000006, + "worldBits": 10.862272556463013, + "surprise": { + "status": "ok", + "bin": 700, + "p": 0.0008458333333332394, + "surprise": 10.20733896308559 + }, + "disagreement": 0.14778939669397495 + }, + "d2": { + "outcome": "valid", + "merged": { + "q1": 0.8466911764705882, + "q5": 1.0063258232235701, + "q9": 1.2533088235294119 + }, + "gain": 0.25, + "worldBits": 12.5514004191889, + "surprise": { + "status": "ok", + "bin": 2500, + "p": 9.714285714284636e-05, + "surprise": 13.329532555244237 + }, + "disagreement": 0.5374126633995484 + }, + "d3": { + "outcome": "untested", + "merged": { + "q1": 0.9198823529411765, + "q5": 1.068937125748503, + "q9": 1.192 + } + } + }, + "judgments": { + "e1": { + "d1": "confirmed", + "d2": "denied" + }, + "e2": { + "d1": "denied", + "d2": "confirmed" + }, + "e3": { + "d1": "confirmed", + "d2": "denied" + } + }, + "direction": { + "weights": [ + 0.0, + 0.9527410207939508, + 0.0, + 0.04725897920604919 + ], + "weightsKept": false, + "gains": { + "d1": { + "gain": 0.07000000000000006, + "measured": true + }, + "d2": { + "gain": 0.25, + "measured": true + }, + "d3": { + "gain": -0.04409262759924386, + "measured": false + } + }, + "experiment": "d2", + "gain": 0.25, + "found": true + } + }, + "note": "两个有效实验：权重按两处密度连乘更新；d2 的实测 +0.25 落在 e2 的范围里、其余两条之外。" + }, + { + "name": "settle_no_direction", + "section": "6.12 结算；6.9", + "input": { + "b": 1.0, + "direction": "maximize", + "lambda": 0.1, + "experiments": [ + "d1", + "d2", + "d3" + ], + "explanations": [ + { + "id": "e1", + "author": "researcher-1", + "ranges": { + "d1": { + "q1": 1.02, + "q5": 1.05, + "q9": 1.08 + }, + "d2": { + "q1": 0.95, + "q5": 1.0, + "q9": 1.05 + }, + "d3": { + "q1": 1.0, + "q5": 1.1, + "q9": 1.2 + } + } + }, + { + "id": "e2", + "author": "researcher-2", + "ranges": { + "d1": { + "q1": 1.0, + "q5": 1.03, + "q9": 1.06 + }, + "d2": { + "q1": 1.1, + "q5": 1.2, + "q9": 1.3 + }, + "d3": { + "q1": 0.9, + "q5": 0.95, + "q9": 1.0 + } + } + }, + { + "id": "e3", + "author": "researcher-2", + "ranges": { + "d1": { + "q1": 0.98, + "q5": 1.04, + "q9": 1.1 + }, + "d2": { + "q1": 0.8, + "q5": 0.9, + "q9": 1.0 + }, + "d3": { + "q1": 1.05, + "q5": 1.1, + "q9": 1.15 + } + } + } + ], + "epsilon": 0.01, + "outcomes": { + "d1": { + "kind": "valid", + "score": 0.97 + }, + "d2": { + "kind": "valid", + "score": 0.8 + }, + "d3": { + "kind": "valid", + "score": 0.9 + } + } + }, + "expected": { + "experiments": { + "d1": { + "outcome": "valid", + "merged": { + "q1": 0.992, + "q5": 1.0428015564202335, + "q9": 1.0922077922077922 + }, + "gain": -0.030000000000000027, + "worldBits": 10.862272556463013, + "surprise": { + "status": "ok", + "bin": -300, + "p": 0.00013333333333331854, + "surprise": 12.872674880270766 + }, + "disagreement": 0.14778939669397495 + }, + "d2": { + "outcome": "valid", + "merged": { + "q1": 0.8466911764705882, + "q5": 1.0063258232235701, + "q9": 1.2533088235294119 + }, + "gain": -0.19999999999999996, + "worldBits": 12.5514004191889, + "surprise": { + "status": "ok", + "bin": -2000, + "p": 9.714285714284646e-05, + "surprise": 13.329532555244235 + }, + "disagreement": 0.5374126633995484 + }, + "d3": { + "outcome": "valid", + "merged": { + "q1": 0.9198823529411765, + "q5": 1.068937125748503, + "q9": 1.192 + }, + "gain": -0.09999999999999998, + "worldBits": 12.165720869087144, + "surprise": { + "status": "ok", + "bin": -1000, + "p": 0.00023611111111108532, + "surprise": 12.048246444854218 + }, + "disagreement": 0.19153258930906247 + } + }, + "judgments": { + "e1": { + "d1": "denied", + "d2": "denied", + "d3": "denied" + }, + "e2": { + "d1": "denied", + "d2": "denied", + "d3": "confirmed" + }, + "e3": { + "d1": "denied", + "d2": "confirmed", + "d3": "denied" + } + }, + "direction": { + "weights": [ + 0.0, + 0.0, + 0.0, + 1.0 + ], + "weightsKept": false, + "gains": { + "d1": { + "gain": -0.030000000000000027, + "measured": true + }, + "d2": { + "gain": -0.19999999999999996, + "measured": true + }, + "d3": { + "gain": -0.09999999999999998, + "measured": true + } + }, + "experiment": "d1", + "gain": -0.030000000000000027, + "found": false + } + }, + "note": "全部实验的收益都不为正（−0.03、−0.2、−0.1）：最大收益 −0.03 ≤ 0，记没有指出能涨分的方向。" + }, + { + "name": "settle_all_insufficient", + "section": "6.12 结算；6.9", + "input": { + "b": 1.0, + "direction": "maximize", + "lambda": 0.1, + "experiments": [ + "d1", + "d2", + "d3" + ], + "explanations": [ + { + "id": "e1", + "author": "researcher-1", + "ranges": { + "d1": { + "q1": 1.02, + "q5": 1.05, + "q9": 1.08 + }, + "d2": { + "q1": 0.95, + "q5": 1.0, + "q9": 1.05 + }, + "d3": { + "q1": 1.0, + "q5": 1.1, + "q9": 1.2 + } + } + }, + { + "id": "e2", + "author": "researcher-2", + "ranges": { + "d1": { + "q1": 1.0, + "q5": 1.03, + "q9": 1.06 + }, + "d2": { + "q1": 1.1, + "q5": 1.2, + "q9": 1.3 + }, + "d3": { + "q1": 0.9, + "q5": 0.95, + "q9": 1.0 + } + } + }, + { + "id": "e3", + "author": "researcher-2", + "ranges": { + "d1": { + "q1": 0.98, + "q5": 1.04, + "q9": 1.1 + }, + "d2": { + "q1": 0.8, + "q5": 0.9, + "q9": 1.0 + }, + "d3": { + "q1": 1.05, + "q5": 1.1, + "q9": 1.15 + } + } + } + ], + "epsilon": 0.01, + "outcomes": { + "d1": { + "kind": "insufficient" + }, + "d2": { + "kind": "insufficient" + }, + "d3": { + "kind": "insufficient" + } + } + }, + "expected": { + "experiments": { + "d1": { + "outcome": "insufficient", + "merged": { + "q1": 0.992, + "q5": 1.0428015564202335, + "q9": 1.0922077922077922 + } + }, + "d2": { + "outcome": "insufficient", + "merged": { + "q1": 0.8466911764705882, + "q5": 1.0063258232235701, + "q9": 1.2533088235294119 + } + }, + "d3": { + "outcome": "insufficient", + "merged": { + "q1": 0.9198823529411765, + "q5": 1.068937125748503, + "q9": 1.192 + } + } + }, + "judgments": { + "e1": {}, + "e2": {}, + "e3": {} + }, + "direction": { + "weights": [ + 0.45, + 0.225, + 0.225, + 0.1 + ], + "weightsKept": false, + "gains": { + "d1": { + "gain": 0.04225000000000004, + "measured": false + }, + "d2": { + "gain": 0.02750000000000002, + "measured": false + }, + "d3": { + "gain": 0.06375, + "measured": false + } + }, + "experiment": "d3", + "gain": 0.06375, + "found": true + } + }, + "note": "没有一个有效实验：不判、不记账，权重保持起始权重，方向按起始权重估的期望收益。" + }, + { + "name": "settle_outside_all", + "section": "6.12 结算；6.9", + "input": { + "b": 1.0, + "direction": "maximize", + "lambda": 0.1, + "experiments": [ + "d1", + "d2", + "d3" + ], + "explanations": [ + { + "id": "e1", + "author": "researcher-1", + "ranges": { + "d1": { + "q1": 1.02, + "q5": 1.05, + "q9": 1.08 + }, + "d2": { + "q1": 0.95, + "q5": 1.0, + "q9": 1.05 + }, + "d3": { + "q1": 1.0, + "q5": 1.1, + "q9": 1.2 + } + } + }, + { + "id": "e2", + "author": "researcher-2", + "ranges": { + "d1": { + "q1": 1.0, + "q5": 1.03, + "q9": 1.06 + }, + "d2": { + "q1": 1.1, + "q5": 1.2, + "q9": 1.3 + }, + "d3": { + "q1": 0.9, + "q5": 0.95, + "q9": 1.0 + } + } + }, + { + "id": "e3", + "author": "researcher-2", + "ranges": { + "d1": { + "q1": 0.98, + "q5": 1.04, + "q9": 1.1 + }, + "d2": { + "q1": 0.8, + "q5": 0.9, + "q9": 1.0 + }, + "d3": { + "q1": 1.05, + "q5": 1.1, + "q9": 1.15 + } + } + } + ], + "epsilon": 0.01, + "outcomes": { + "d1": { + "kind": "valid", + "score": 3.0 + }, + "d2": { + "kind": "untested" + }, + "d3": { + "kind": "untested" + } + } + }, + "expected": { + "experiments": { + "d1": { + "outcome": "valid", + "merged": { + "q1": 0.992, + "q5": 1.0428015564202335, + "q9": 1.0922077922077922 + }, + "gain": 2.0, + "worldBits": 10.862272556463013, + "surprise": { + "status": "priorZero", + "bin": 20000, + "p": 0.0 + }, + "disagreement": 0.14778939669397495 + }, + "d2": { + "outcome": "untested", + "merged": { + "q1": 0.8466911764705882, + "q5": 1.0063258232235701, + "q9": 1.2533088235294119 + } + }, + "d3": { + "outcome": "untested", + "merged": { + "q1": 0.9198823529411765, + "q5": 1.068937125748503, + "q9": 1.192 + } + } + }, + "judgments": { + "e1": { + "d1": "denied" + }, + "e2": { + "d1": "denied" + }, + "e3": { + "d1": "denied" + } + }, + "direction": { + "weights": [ + 0.45, + 0.225, + 0.225, + 0.1 + ], + "weightsKept": true, + "gains": { + "d1": { + "gain": 2.0, + "measured": true + }, + "d2": { + "gain": 0.02750000000000002, + "measured": false + }, + "d3": { + "gain": 0.06375, + "measured": false + } + }, + "experiment": "d1", + "gain": 2.0, + "found": true + } + }, + "note": "d1 实测 +2.0 落在所有成分（含开放预留）的范围之外，权重无从更新：保持起始权重（weightsKept）。" + }, + { + "name": "minimize_all", + "section": "6.12", + "input": { + "b": 1.0, + "direction": "minimize", + "lambda": 0.1, + "experiments": [ + "d1", + "d2" + ], + "explanations": [ + { + "id": "e1", + "author": "researcher-1", + "ranges": { + "d1": { + "q1": 0.85, + "q5": 0.9, + "q9": 0.95 + }, + "d2": { + "q1": 0.9, + "q5": 1.0, + "q9": 1.1 + } + } + }, + { + "id": "e2", + "author": "researcher-2", + "ranges": { + "d1": { + "q1": 0.8, + "q5": 0.85, + "q9": 0.9 + }, + "d2": { + "q1": 0.95, + "q5": 1.05, + "q9": 1.15 + } + } + } + ], + "balance": 7, + "batchCap": 2, + "maxExperiments": 4, + "u": 0.5, + "decay": 0.5 + }, + "expected": { + "priorGains": { + "d1": 0.125, + "d2": -0.024999999999999994 + }, + "g0": 0.125, + "weights": [ + 0.45, + 0.45, + 0.1 + ], + "combos": [ + { + "experiments": [ + "d1", + "d2" + ], + "voi": 0.010397222222222214, + "rHat": 3, + "mPrime": 1.75, + "value": 0.009097569444444436, + "cost": 0.020833333333333332, + "net": -0.011735763888888896 + } + ], + "diagnose": false, + "chosen": [], + "chosenNet": 0.0 + }, + "note": "越小越好：收益 = b − x。d1 两条解释都预测降到 0.9 左右（收益约 +0.1），d2 预测上升。" + }, + { + "name": "settle_minimize", + "section": "6.12 结算；6.9", + "input": { + "b": 1.0, + "direction": "minimize", + "lambda": 0.1, + "experiments": [ + "d1", + "d2" + ], + "explanations": [ + { + "id": "e1", + "author": "researcher-1", + "ranges": { + "d1": { + "q1": 0.85, + "q5": 0.9, + "q9": 0.95 + }, + "d2": { + "q1": 0.9, + "q5": 1.0, + "q9": 1.1 + } + } + }, + { + "id": "e2", + "author": "researcher-2", + "ranges": { + "d1": { + "q1": 0.8, + "q5": 0.85, + "q9": 0.9 + }, + "d2": { + "q1": 0.95, + "q5": 1.05, + "q9": 1.15 + } + } + } + ], + "epsilon": 0.01, + "outcomes": { + "d1": { + "kind": "valid", + "score": 0.88 + }, + "d2": { + "kind": "valid", + "score": 1.05 + } + } + }, + "expected": { + "experiments": { + "d1": { + "outcome": "valid", + "merged": { + "q1": 0.8042553191489362, + "q5": 0.875, + "q9": 0.9457446808510638 + }, + "gain": 0.12, + "worldBits": 11.270948490815508, + "surprise": { + "status": "ok", + "bin": -1200, + "p": 0.0007399999999999179, + "surprise": 10.400187108808021 + }, + "disagreement": 0.2390359525563106 + }, + "d2": { + "outcome": "valid", + "merged": { + "q1": 0.8964285714285714, + "q5": 1.025, + "q9": 1.1535714285714285 + }, + "gain": -0.050000000000000044, + "worldBits": 12.14559114140212, + "surprise": { + "status": "ok", + "bin": 500, + "p": 0.0003711111111110703, + "surprise": 11.39586118340523 + }, + "disagreement": 0.11951797627815353 + } + }, + "judgments": { + "e1": { + "d1": "confirmed", + "d2": "confirmed" + }, + "e2": { + "d1": "confirmed", + "d2": "confirmed" + } + }, + "direction": { + "weights": [ + 0.4961715160796319, + 0.496171516079633, + 0.0076569678407350855 + ], + "weightsKept": false, + "gains": { + "d1": { + "gain": 0.12, + "measured": true + }, + "d2": { + "gain": -0.050000000000000044, + "measured": true + } + }, + "experiment": "d1", + "gain": 0.12, + "found": true + } + }, + "note": "越小越好：d1 实测 0.88，收益 = 1.0 − 0.88 = 0.12；d2 实测 1.05，收益 −0.05。" + } + ] +} diff --git a/tests/fixtures/rules/gen_diagnosis.py b/tests/fixtures/rules/gen_diagnosis.py new file mode 100644 index 0000000..746c937 --- /dev/null +++ b/tests/fixtures/rules/gen_diagnosis.py @@ -0,0 +1,318 @@ +"""生成诊断（方法.md 6.12）的预期值 JSON（diagnosis.json）。 + +用法（在仓库根目录）： + python3 tests/fixtures/rules/gen_diagnosis.py +只用标准库；依赖同目录的 rules_ref.py 里的基础件（折线、有理数分布、Stieltjes 精确积分、比特、惊讶、分歧量）。 +重跑结果逐位相同。 + +独立性：诊断的成分模型、方向收益、信息价值、价值减成本、选实验、结算都在本文件里只照 docs/方法.md 6.12 和 6.7、6.9 +的文字写成，没有读过 src/rules/diagnosis.ts 和 src/rules/value.ts。 + +算法（有理数精确，浮点输入先精确转成有理数）： +- 每个成分每个实验的分布：各条解释的折线加这个实验自己的开放预留（6.3"诊断"），换成收益 g = 改进量（x − b，越小越好时 b − x）； +- E(g_d)：各成分期望按起始权重平均； +- 全做（k = 实验总数）的 VOI：Σ_j w_j · E_j[max(0, max_d g_d)]（rules_ref.expect_max 的 floor0 模式）− max(0, max_d E g_d)； +- 只做一个实验 d0 的 VOI（exactK1）：d0 的各成分在相邻折点之间密度是常数，所以后验权重在每一段里是常数， + 没做的实验的期望收益 c_e 在每一段里是常数，max(0, v, c_e…) 对 v 的积分有闭式； +- 价值 = u·m′·VOI，R̂ = ⌈(余额 − 1) ÷ K⌉，成本 = 2·max(0, ĝ₀)·[(K+1)/(K+2) − K/(K+1)]； +- 推出的方向：成分新权重 ∝ 起始权重 × 各有效实验在实测处的密度（所在段斜率，段左闭右开）。 + +生成的文件：diagnosis.json，格式： + {"generator", "tolerance", "scenarios": [{"name", "section", "input", "expected", "note"}]} +- 场景 input：{"b", "direction", "lambda", "experiments": ["d1", ...], + "explanations": [{"id", "author", "ranges": {"d1": {"q1","q5","q9"}, ...}}]}， + 选实验场景另带 "balance"、"batchCap"、"maxExperiments"、"u"、"decay"，结算场景另带 "epsilon" 和 "outcomes"。 +- 分数一律是原值；收益、VOI 是收益刻度（改进量）。 +""" + +import json +import math +import os +import sys +from fractions import Fraction as Fr +from itertools import combinations + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import rules_ref as R # noqa: E402 + +HERE = os.path.dirname(os.path.abspath(__file__)) +F = float + + +def tri(t): + return {"q1": t[0], "q5": t[1], "q9": t[2]} + + +def ids_key(s): + return R.factor_key(s) + + +class Model: + """诊断的成分模型（有理数）。explanations 按编号排好；成分 = 各解释 + 开放预留。""" + + def __init__(self, b, direction, lam, experiments, explanations): + self.b, self.direction, self.lam = fr(b), direction, fr(lam) + self.exps = sorted(experiments, key=ids_key) + self.expl = sorted(explanations, key=lambda e: ids_key(e["id"])) + authors = [e["author"] for e in self.expl] + self.w = R.explanation_weights(authors, lam) + [fr(lam)] + # raw[d][j]：分数原值上的点列；gain[d][j]：收益（改进量）上的点列 + self.raw, self.gain = {}, {} + for d in self.exps: + pts = [R.polyline(*e["ranges"][d]) for e in self.expl] + lo, hi = R.reserve_bounds(pts) + pts.append(R.uniform_pts(lo, hi)) + self.raw[d] = pts + self.gain[d] = [R.improvement_pts(p, self.b, direction) for p in pts] + self.mu = {d: [R.mean(R.single(p)) for p in self.gain[d]] for d in self.exps} + self.prior = {d: sum((wj * m for wj, m in zip(self.w, self.mu[d])), Fr(0)) for d in self.exps} + + def g0(self): + return max(Fr(0), max(self.prior.values())) + + def voi_all(self): + total = Fr(0) + for j, wj in enumerate(self.w): + if wj == 0: + continue + total += wj * R.expect_max([R.single(self.gain[d][j]) for d in self.exps], floor0=True) + return total - self.g0() + + def voi_k1(self, d0): + """只做 d0：每一段里后验权重和没做的实验的期望收益都是常数，对 max(0, v, c…) 精确积分。""" + bps = sorted(set().union(*[set(x for x, _ in p) for p in self.gain[d0]])) + total = Fr(0) + for a, b in zip(bps, bps[1:]): + dens = [R.pts_left(p, b) - R.pts_left(p, a) for p in self.gain[d0]] + dens = [x / (b - a) for x in dens] + marg = sum((wj * x for wj, x in zip(self.w, dens)), Fr(0)) + if marg == 0: + continue + c0 = Fr(0) + for e in self.exps: + if e == d0: + continue + num = sum((wj * x * m for wj, x, m in zip(self.w, dens, self.mu[e])), Fr(0)) + c0 = max(c0, num / marg) + t = min(max(c0, a), b) + total += marg * (c0 * (t - a) + (b * b - t * t) / 2) + return total - self.g0() + + def mixture(self, d): + """实验 d 的成分混合（分数原值）。""" + return R.Dist(list(zip(self.w, self.raw[d]))) + + +def fr(x): + return R.fr(x) + + +def model_of(inp): + expl = [{"id": e["id"], "author": e["author"], "ranges": {d: (r["q1"], r["q5"], r["q9"]) for d, r in e["ranges"].items()}} for e in inp["explanations"]] + return Model(inp["b"], inp["direction"], inp["lambda"], inp["experiments"], expl) + + +def explanation(eid, author, ranges): + return {"id": eid, "author": author, "ranges": {d: tri(t) for d, t in ranges.items()}} + + +def m_prime(balance, K, r): + R_ = -(-(balance - 1) // K) + return R_, sum((fr(r) ** j for j in range(R_)), Fr(0)) + + +def cost_of(g0, K): + return 2 * g0 * (Fr(K + 1, K + 2) - Fr(K, K + 1)) + + +S = [] + + +def select_case(name, inp, balance, K, D, u, r, note): + inp = dict(inp, balance=balance, batchCap=K, maxExperiments=D, u=u, decay=r) + m = model_of(inp) + rows = [] + if balance >= 2: + size = min(len(m.exps), D) + R_, mp = m_prime(balance, K, r) + c = cost_of(m.g0(), K) + for combo in combinations(m.exps, size): + if size == len(m.exps): + voi = m.voi_all() + elif size == 1: + voi = m.voi_k1(combo[0]) + else: + voi = None # 网格近似，没有精确值，只检查结构 + if voi is None: + rows.append({"experiments": list(combo), "voi": None}) + continue + value = fr(u) * mp * voi + rows.append({"experiments": list(combo), "voi": F(voi), "rHat": R_, "mPrime": F(mp), "value": F(value), + "cost": F(c), "net": F(value - c)}) + expected = { + "priorGains": {d: F(m.prior[d]) for d in m.exps}, + "g0": F(m.g0()), + "weights": [F(x) for x in m.w], + "combos": rows, + } + if all(r_["voi"] is not None for r_ in rows) and rows: + eligible = [r_ for r_ in rows if r_["net"] > 1e-12] + if eligible: + top = max(r_["net"] for r_ in eligible) + near = [r_ for r_ in eligible if r_["net"] >= top - 1e-12] + best = min(near, key=lambda r_: [ids_key(x) for x in r_["experiments"]]) + expected["diagnose"] = True + expected["chosen"] = best["experiments"] + expected["chosenNet"] = best["net"] + else: + expected["diagnose"] = False + expected["chosen"] = [] + expected["chosenNet"] = 0.0 + elif not rows: + expected["diagnose"] = False + expected["chosen"] = [] + expected["chosenNet"] = 0.0 + S.append({"name": name, "section": "6.12", "input": inp, "expected": expected, "note": note}) + + +def settle_case(name, inp, outcomes, note): + inp = dict(inp, outcomes=outcomes) + m = model_of(inp) + eps = fr(inp["epsilon"]) + delta = F(eps / 100) + b = F(m.b) + scores = {d: (o["score"] if o["kind"] == "valid" else None) for d, o in outcomes.items()} + rows = {} + for d in m.exps: + dist = m.mixture(d) + row = {"outcome": outcomes[d]["kind"]} + row["merged"] = {"q1": F(R.quantile(dist, Fr(1, 10))), "q5": F(R.quantile(dist, Fr(1, 2))), "q9": F(R.quantile(dist, Fr(9, 10)))} + if scores[d] is not None: + x = scores[d] + row["gain"] = F(R.improve(x, m.b, m.direction)) + row["worldBits"] = R.world_bits(dist, b, delta) + sp = R.surprise(dist, x, b, delta) + row["surprise"] = sp + persons = {} + for e in m.expl: + persons.setdefault(e["author"], []).append(R.polyline(*e["ranges"][d])) + person_dists = [R.Dist([(Fr(1, len(ps)), p) for p in ps]) for _, ps in sorted(persons.items(), key=lambda kv: ids_key(kv[0]))] + row["disagreement"] = R.disagreement(person_dists, b, delta)[0] + rows[d] = row + # 确认与否定：只有有效实验判，落在 [q1, q9]（含两端）确认。 + judgments = {} + for e in m.expl: + judgments[e["id"]] = {d: ("confirmed" if e["ranges"][d][0] <= scores[d] <= e["ranges"][d][2] else "denied") + for d in m.exps if scores[d] is not None} + # 诊断推出的方向 + new = [] + for j, wj in enumerate(m.w): + v = wj + for d in m.exps: + if scores[d] is not None: + v *= R.pts_density(m.raw[d][j], fr(scores[d])) + new.append(v) + tot = sum(new, Fr(0)) + kept = tot == 0 + w2 = m.w if kept else [v / tot for v in new] + gains = {} + for d in m.exps: + if scores[d] is not None: + gains[d] = {"gain": F(R.improve(scores[d], m.b, m.direction)), "measured": True} + else: + gains[d] = {"gain": F(sum((wj * mu for wj, mu in zip(w2, m.mu[d])), Fr(0))), "measured": False} + best = m.exps[0] + for d in m.exps[1:]: + if gains[d]["gain"] > gains[best]["gain"]: + best = d + expected = {"experiments": rows, "judgments": judgments, + "direction": {"weights": [F(x) for x in w2], "weightsKept": kept, "gains": gains, + "experiment": best, "gain": gains[best]["gain"], "found": gains[best]["gain"] > 0}} + S.append({"name": name, "section": "6.12 结算；6.9", "input": inp, "expected": expected, "note": note}) + + +# ------------------------------------------------------------------ 场景 + +def base(explanations, experiments, b=0.0, direction="maximize", lam=0.1): + return {"b": b, "direction": direction, "lambda": lam, "experiments": experiments, "explanations": explanations} + + +# 1. 两条解释对同一个实验预测相反：A 说涨 0.2 左右，B 说跌 0.2 左右。 +OPP = [explanation("e1", "researcher-1", {"d1": (0.15, 0.2, 0.25)}), + explanation("e2", "researcher-2", {"d1": (-0.25, -0.2, -0.15)})] +select_case("opposite_lambda0", base(OPP, ["d1"], lam=0.0), 7, 2, 4, 0.5, 0.5, + "λ = 0，两个成分的范围不相交：做了就知道是哪一条。手算：做了以后 E[max(0, g)] = 0.5 × 0.2 = 0.1，" + "不做时 max(0, E g) = max(0, 0.5×0.2 + 0.5×(−0.2)) = 0，VOI = 0.1。") +select_case("opposite_lambda01", base(OPP, ["d1"]), 7, 2, 4, 0.5, 0.5, "同上，λ = 0.1，开放预留铺在两端之外。") + +# 2. 两条解释预测一样：λ = 0 时信息价值为 0；λ = 0.1 时只剩开放预留带来的一点。 +SAME = [explanation("e1", "researcher-1", {"d1": (0.1, 0.2, 0.3)}), + explanation("e2", "researcher-2", {"d1": (0.1, 0.2, 0.3)})] +select_case("same_lambda0", base(SAME, ["d1"], lam=0.0), 7, 2, 4, 0.5, 0.5, + "λ = 0，两条解释一样，做不做都不改变看法：VOI = 0（手算：g 的取值范围 0 到 0.4 全在 0 以上，E[max(0, g)] = E g = 0.2）。净值 = −成本 < 0，不诊断。") +select_case("same_lambda01", base(SAME, ["d1"]), 7, 2, 4, 0.5, 0.5, "λ = 0.1：解释们的范围都在 0 以上，只有开放预留（铺在 −0.2 到 0.6）能给出负的结果，所以 VOI 只剩预留带来的一小部分。") + +# 3. 三个实验、三条解释，全做（3 ≤ D）。 +MULTI = [explanation("e1", "researcher-1", {"d1": (0.02, 0.05, 0.08), "d2": (-0.05, 0.0, 0.05), "d3": (0.0, 0.1, 0.2)}), + explanation("e2", "researcher-2", {"d1": (0.0, 0.03, 0.06), "d2": (0.1, 0.2, 0.3), "d3": (-0.1, -0.05, 0.0)}), + explanation("e3", "researcher-3", {"d1": (-0.02, 0.04, 0.1), "d2": (-0.2, -0.1, 0.0), "d3": (0.05, 0.1, 0.15)})] +select_case("multi_all", base(MULTI, ["d1", "d2", "d3"]), 9, 2, 4, 0.5, 0.5, + "三个实验不超过 D = 4，只有全做一个组合。R̂ = ⌈8 ÷ 2⌉ = 4，m′ = 1.875，成本 = 2ĝ₀ × (3/4 − 2/3)。") + +# 4. 实验数超过 D：只比较装满的组合。D = 1 时是三个单实验组合（有精确值）；D = 2 时只有三个两实验组合（只查结构）。 +select_case("over_cap_d1", base(MULTI, ["d1", "d2", "d3"]), 9, 2, 1, 0.5, 0.5, + "三个实验超过 D = 1：只有三个单实验组合，没有空组合；精确值用逐段闭式积分。") +select_case("over_cap_d2", base(MULTI, ["d1", "d2", "d3"]), 9, 2, 2, 0.5, 0.5, + "三个实验超过 D = 2：只有三个两实验组合，没有单实验组合；只检查组合列表（网格近似没有精确值）。") + +# 5. 余额不到 2 次不诊断。 +select_case("balance_one", base(MULTI, ["d1", "d2", "d3"]), 1, 2, 4, 0.5, 0.5, "余额 1：没有可选组合，不诊断。") + +# 6. 平票按实验编号：d2 和 d10 的预测完全一样，D = 1 时两个单实验组合净值相同，取 d2（自然序，d2 排在 d10 前面）。 +TIE = [explanation("e1", "researcher-1", {"d2": (-0.2, 0.0, 0.2), "d10": (-0.2, 0.0, 0.2)}), + explanation("e2", "researcher-2", {"d2": (0.1, 0.3, 0.5), "d10": (0.1, 0.3, 0.5)})] +select_case("tie_by_id", base(TIE, ["d10", "d2"]), 7, 2, 1, 0.5, 0.5, "两个实验的预测完全一样，净值相同（差不到 1e-12），取实验编号小的 d2。") + +# 7. 结算。 +def shifted(expls, b): + """把以基线为 0 的改进量范围搬成分数原值（基线 b）。""" + return [dict(e, ranges={d: {k: v + b for k, v in r.items()} for d, r in e["ranges"].items()}) for e in expls] + + +SETTLE = shifted(MULTI[:2] + [explanation("e3", "researcher-2", {"d1": (-0.02, 0.04, 0.1), "d2": (-0.2, -0.1, 0.0), "d3": (0.05, 0.1, 0.15)})], 1.0) +SINP = dict(base(SETTLE, ["d1", "d2", "d3"], b=1.0), epsilon=0.01) +settle_case("settle_mixed", SINP, + {"d1": {"kind": "valid", "score": 1.045}, "d2": {"kind": "insufficient"}, "d3": {"kind": "untested"}}, + "d1 有效，d2 证据不足（不判、不记账），d3 没做。研究者 2 交了两条解释，各占他那份的一半。") +settle_case("settle_two_valid", SINP, + {"d1": {"kind": "valid", "score": 1.07}, "d2": {"kind": "valid", "score": 1.25}, "d3": {"kind": "untested"}}, + "两个有效实验：权重按两处密度连乘更新；d2 的实测 +0.25 落在 e2 的范围里、其余两条之外。") +settle_case("settle_no_direction", dict(SINP), + {"d1": {"kind": "valid", "score": 0.97}, "d2": {"kind": "valid", "score": 0.8}, "d3": {"kind": "valid", "score": 0.9}}, + "全部实验的收益都不为正（−0.03、−0.2、−0.1）：最大收益 −0.03 ≤ 0，记没有指出能涨分的方向。") +settle_case("settle_all_insufficient", SINP, + {"d1": {"kind": "insufficient"}, "d2": {"kind": "insufficient"}, "d3": {"kind": "insufficient"}}, + "没有一个有效实验：不判、不记账，权重保持起始权重，方向按起始权重估的期望收益。") +settle_case("settle_outside_all", SINP, + {"d1": {"kind": "valid", "score": 3.0}, "d2": {"kind": "untested"}, "d3": {"kind": "untested"}}, + "d1 实测 +2.0 落在所有成分（含开放预留）的范围之外，权重无从更新：保持起始权重（weightsKept）。") + +MIN_EXPL = [explanation("e1", "researcher-1", {"d1": (0.95, 0.9, 0.85)[::-1], "d2": (0.9, 1.0, 1.1)}), + explanation("e2", "researcher-2", {"d1": (0.8, 0.85, 0.9), "d2": (0.95, 1.05, 1.15)})] +MIN_INP = dict(base(MIN_EXPL, ["d1", "d2"], b=1.0, direction="minimize"), epsilon=0.01) +select_case("minimize_all", dict(base(MIN_EXPL, ["d1", "d2"], b=1.0, direction="minimize")), 7, 2, 4, 0.5, 0.5, + "越小越好：收益 = b − x。d1 两条解释都预测降到 0.9 左右（收益约 +0.1），d2 预测上升。") +settle_case("settle_minimize", MIN_INP, {"d1": {"kind": "valid", "score": 0.88}, "d2": {"kind": "valid", "score": 1.05}}, + "越小越好：d1 实测 0.88，收益 = 1.0 − 0.88 = 0.12；d2 实测 1.05，收益 −0.05。") + +doc = {"generator": "gen_diagnosis.py", + "tolerance": { + "exact": "有理数精确算出的量（成分权重、事前期望收益、ĝ₀、全做的 VOI、R̂、m′、成本、价值、净值、收益、更新后的权重）：绝对误差 1e-12", + "exactK1": "只做一个实验的 VOI 是逐段闭式积分，实现用部分格网格近似：|差| ≤ 1e-9 + 2e-4·|精确值|", + "bits": "世界比特、惊讶、分歧量是浮点：绝对误差 1e-9；合并范围是分位点：绝对误差 1e-9", + "discrete": "是否诊断、选中的实验、确认与否定、有没有指出方向、实验的结局：必须完全相同"}, + "scenarios": S} +with open(os.path.join(HERE, "diagnosis.json"), "w", encoding="utf-8") as f: + json.dump(doc, f, ensure_ascii=False, indent=1) + f.write("\n") +print(len(S), "个场景") diff --git a/tests/flow-diagnosis-docker.test.ts b/tests/flow-diagnosis-docker.test.ts new file mode 100644 index 0000000..e7dd941 --- /dev/null +++ b/tests/flow-diagnosis-docker.test.ts @@ -0,0 +1,230 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test, type TestContext } from "node:test"; +import { stepList } from "../src/config/index.ts"; +import { fullConfig as checkedFull } from "../src/flow/frozen.ts"; +import { judgeDiagnosisRun } from "../src/flow/formal-run.ts"; +import { containerName, freshDir, runPaths, workspaceRepo } from "../src/flow/paths.ts"; +import { recover } from "../src/flow/recover.ts"; +import { diagnosisRun, diagnosisSettle } from "../src/flow/steps/diagnosis.ts"; +import type { Study } from "../src/flow/study.ts"; +import { Ledger } from "../src/ledger/index.ts"; +import { Board } from "../src/materials/index.ts"; +import { runBalance, settleDiagnosis, type SettleDiagnosisInput } from "../src/rules/index.ts"; +import { Runner } from "../src/runner/index.ts"; +import { diagnosisRun as diagnosisDefinition, diagnosisUnversionedNames, loadTask, prepareDiagnosisRun, type DiagnosisJudged } from "../src/tasks/index.ts"; +import { checkoutVersion, createRepository, saveVersion } from "../src/workspace/index.ts"; +import { fullConfig } from "./config-helpers.ts"; +import { judged, r, Writer } from "./materials-helpers.ts"; + +// 诊断运行在 flow 里的几处，用真实 Docker 测，不花模型的钱（任务书 18 第 1.6 节）： +// 选中的实验拿掉一个后跑的只是剩下的实验、逐个实验判分和结算；诊断运行恢复只核对、不重做、只扣一次。 +// 镜像一律 inforsi-test-flow-dx 开头的固定名字，只动自己前缀的。判分的预期值是手算的相切布局：26 个半径 1/64 的圆排成一行， +// 圆心间距 1/32，半径和 26/64 = 0.40625（同 tests/tasks-diagnosis.test.ts，都是 2 的幂有理数，没有舍入）。 + +const TASK_DIR = join(import.meta.dirname, "..", "tasks", "circle_packing"); +const BASELINE_SUM = 26 / 64; + +function docker(args: string[]): { status: number | null; stderr: string } { + return spawnSync("docker", args, { encoding: "utf8" }); +} + +function removeImage(tag: string): void { + const removed = docker(["rmi", tag]); + if (removed.status !== 0 && !removed.stderr.includes("No such image")) throw new Error(removed.stderr); +} + +function tempDir(t: TestContext): string { + const dir = mkdtempSync(join(tmpdir(), "inforsi-flow-dx-")); + t.after(() => rmSync(dir, { recursive: true, force: true })); + return dir; +} + +/** 诊断程序：d1 原样交回基线的原始答案；别的实验编号被调用就崩（用来证明拿掉的实验没被调用）。 */ +const PROGRAM = `import json + + +def run_experiment(experiment_id, baseline_dir): + if experiment_id == "d1": + with open(baseline_dir + "/solution.json") as handle: + solution = json.load(handle) + return solution["centers"], solution["radii"], solution["reported_sum"] + raise RuntimeError("不该被调用的实验 " + experiment_id) +`; + +/** 真实任务镜像、账本、裸仓库；第 0 轮基线已跑完（运行目录里放着相切布局的原始答案，半径和 0.40625），诊断程序已存成版本。 */ +async function setup(t: TestContext, suffix: string, maxRepairs: number): Promise<{ s: Study; w: Writer; version: string }> { + const dir = tempDir(t); + const id = `test-flow-dx-${suffix}`; + removeImage(`inforsi-${id}`); + t.after(() => removeImage(`inforsi-${id}`)); + const task = loadTask(TASK_DIR); + const image = await new Runner(null).buildImage(task.dir, `inforsi-${id}`); + const ledger = Ledger.open(join(dir, "ledger.sqlite")); + t.after(() => ledger.close()); + const config = { ...fullConfig(), studyId: id, maxRepairs }; + if (config.flow.preset !== "full") throw new Error("测试配置应是完整流程"); + config.flow = { ...config.flow, diagnosis: { enabled: true, perResearcher: 2, maxExperiments: 4 }, ablation: { enabled: false } }; + const s = { dir, task, runner: new Runner(null), ledger, stop: new AbortController().signal, + frozen: { config: checkedFull(config), steps: stepList(config), facts: task.facts, prompts: null, image, seed: 1 } } as unknown as Study; + const w = new Writer(ledger); + ledger.append("code", { object: "base", version: "v-base", locations: null }); + w.run("base-1", "baseline", 0, null, judged(BASELINE_SUM, "main")); + const baseRun = runPaths(dir, "base-1").run; + mkdirSync(baseRun, { recursive: true }); + writeFileSync(join(baseRun, "solution.json"), JSON.stringify({ + centers: Array.from({ length: 26 }, (_, i) => [7 / 64 + i / 32, 0.5]), + radii: Array(26).fill(1 / 64), + reported_sum: 99, + })); + createRepository(workspaceRepo(dir)); + const program = join(dir, "program"); + mkdirSync(program); + writeFileSync(join(program, "initial.py"), PROGRAM); + const saved = saveVersion(workspaceRepo(dir), program, diagnosisUnversionedNames(task)); + if (!saved.saved) throw new Error(`诊断程序存不成版本：${saved.reason}`); + return { s, w, version: saved.version }; +} + +const RANGES = { + a1: [r(0.5, 0.6, 0.7), r(0.3, 0.4, 0.45)], + a2: [r(0.2, 0.3, 0.4), r(0.3, 0.4, 0.45)], +} as const; + +/** 本轮诊断写到"选中 d1、d2，核验只有 d1 通过"为止：诊断解释与实验、判重合并、诊断预测、选实验、诊断程序、核验。 */ +function selectedAndVerified({ w, version }: { w: Writer; version: string }): void { + const experiment = (name: string) => ({ name, spot: `${name}的位置`, body: `${name}的做法`, metrics: [] }); + w.open("r1-diagnosisProposal", "diagnosisProposal", 1, ["a1", "a2"]); + for (const [author, name] of [["a1", "实验甲"], ["a2", "实验乙"]] as const) { + w.sub("r1-diagnosisProposal", author, "diagnosisProposal", null, null, { explanations: [{ body: `${author}的解释` }], experiments: [experiment(name)] }); + } + w.close("r1-diagnosisProposal"); + w.decision("diagnosisDedup", 1, null, { + experiments: [{ experiment: "d1", sources: [{ author: "a1", index: 0 }] }, { experiment: "d2", sources: [{ author: "a2", index: 0 }] }], + }); + w.open("r1-diagnosisPrediction", "diagnosisPrediction", 1, ["a1", "a2"]); + for (const author of ["a1", "a2"] as const) { + w.sub("r1-diagnosisPrediction", author, "diagnosisPrediction", null, null, { + predictions: ["d1", "d2"].map((d, i) => ({ explanation: `r1-${author}-dx1`, experiment: d, range: RANGES[author][i]! })), + body: "理由", + }); + } + w.close("r1-diagnosisPrediction"); + w.decision("selectDiagnosis", 1, null, { diagnose: true, chosen: ["d1", "d2"] }); + w.l.append("code", { object: "r1-diagnosis", version, locations: [] }); + w.open("r1-diagnosisVerification-1", "diagnosisVerification", 1, ["verifier-r1-diagnosis"]); + w.sub("r1-diagnosisVerification-1", "verifier-r1-diagnosis", "diagnosisVerification", "r1-diagnosis", null, { + experiments: [ + { experiment: "d1", followed: true, nothingElse: true, where: "initial.py:4" }, + { experiment: "d2", followed: false, nothingElse: true, where: "没找到" }, + ], + body: "无", + }); + w.close("r1-diagnosisVerification-1"); +} + +test("诊断运行：核验没通过又没有修复次数的实验拿掉，只跑剩下的实验；逐个实验判分，结算里拿掉的记证据不足", { timeout: 180_000 }, async (t) => { + const fixture = await setup(t, "settle", 0); + const { s } = fixture; + selectedAndVerified(fixture); + t.after(() => { docker(["rm", "-f", containerName(s.frozen.config.studyId, "r1-diagnosis-1")]); }); + assert.equal(await diagnosisRun(s, 1, "r1-diagnosis-1"), "done"); + const records = s.ledger.records(); + const planned = records.find((x) => x.kind === "runPlanned" && x.content.purpose === "diagnosis")!; + if (planned.kind !== "runPlanned") throw new Error("缺登记"); + assert.deepEqual({ round: planned.content.round, object: planned.content.object }, { round: 1, object: "r1-diagnosis" }); + // 只调 d1，总时限 120 秒由它一个实验独享。 + const { command } = planned.content.definition; + assert.deepEqual(command.slice(command.indexOf("--experiments")), ["--experiments", "d1"]); + assert.equal(command[command.indexOf("--seconds-each") + 1], "120"); + assert.equal(records.filter((x) => x.kind === "runStarted" && x.content.run === "r1-diagnosis-1").length, 1); + const result = records.findLast((x) => x.kind === "runResult")!; + if (result.kind !== "runResult") throw new Error("缺结果"); + assert.equal(result.content.process.ending, "exited"); + assert.equal(result.content.judged, null); + assert.deepEqual(runBalance(13, new Board(records).runStates()), { charged: 2, reserved: 0, balance: 11 }); + + await diagnosisSettle(s, 1); + const settled = s.ledger.records().find((x) => x.kind === "diagnosisSettlement")!; + if (settled.kind !== "diagnosisSettlement") throw new Error("缺结算"); + const input: SettleDiagnosisInput = { + baseline: BASELINE_SUM, + direction: "maximize", + epsilon: s.frozen.facts.epsilon, + lambda: 0.1, + experiments: [ + { experiment: "d1", outcome: { kind: "valid", score: BASELINE_SUM } }, + { experiment: "d2", outcome: { kind: "insufficient" } }, + ], + explanations: (["a1", "a2"] as const).map((author) => ({ + id: `r1-${author}-dx1`, + author, + experiments: ["d1", "d2"].map((experiment, i) => ({ experiment, range: RANGES[author][i]! })), + })), + }; + const { judged: j, dropped, run, ...rest } = settled.content.result as unknown as { judged: DiagnosisJudged; dropped: Array<{ experiment: string; reason: string }>; run: string }; + assert.equal(run, "r1-diagnosis-1"); + assert.deepEqual(rest, { input, output: settleDiagnosis(input) }); + // 判分只有 d1 一项：拿掉的 d2 没跑、没判。 + assert.equal(j.experiments.length, 1); + const d1 = j.experiments[0]!; + assert.deepEqual([d1.id, d1.ending, d1.verdict, d1.score], ["d1", "finished", "valid", { status: "measured", value: BASELINE_SUM }]); + assert.deepEqual(dropped.map((d) => d.experiment), ["d2"]); + assert.match(dropped[0]!.reason, /第 1 次核验仍不通过，修复次数上限 0 已用完：没照写法做；没找到/); + // 结算写进去以后诊断运行公开；每轮最多一条结算。 + assert.ok(new Board(s.ledger.records()).runs().filter((x) => x.plan.purpose === "diagnosis").every((x) => new Board(s.ledger.records()).runPublic(x))); + assert.throws(() => s.ledger.append("diagnosisSettlement", { round: 1, result: null })); +}); + +/** 手工把一次诊断运行做到"登记了"：和 diagnosisRun 同样的目录准备，但不启动（启动由各个测试自己决定）。 */ +function planned(s: Study, version: string): { run: string; name: string; dir: string } { + const run = "r1-diagnosis-1"; + const paths = runPaths(s.dir, run); + const definition = diagnosisDefinition(s.task, s.frozen.image, String(s.frozen.seed), ["d1"]); + s.ledger.append("runPlanned", { run, purpose: "diagnosis", round: 1, object: "r1-diagnosis", definition }); + freshDir(paths.base); + checkoutVersion(workspaceRepo(s.dir), version, freshDir(paths.code)); + prepareDiagnosisRun(s.task, paths.code, runPaths(s.dir, "base-1").run, freshDir(paths.run), [{ id: "d1", metrics: [] }]); + return { run, name: containerName(s.frozen.config.studyId, run), dir: paths.run }; +} + +test("诊断运行恢复：容器已启动、账本没写确认启动和结果，接上原容器不重跑，只扣一次，运行记录不带判分，判分从运行目录重做", { timeout: 180_000 }, async (t) => { + const { s, version } = await setup(t, "recover", 0); + const { run, name, dir } = planned(s, version); + t.after(() => { docker(["rm", "-f", name]); }); + assert.deepEqual(await s.runner.start(name, diagnosisDefinition(s.task, s.frozen.image, String(s.frozen.seed), ["d1"]), dir), { started: true }); + // 中断点：容器已启动，账本只有登记。恢复不调用 diagnosisRun，而是找到原容器等它。 + assert.equal(await recover(s), null); + const records = s.ledger.records(); + assert.deepEqual(["runPlanned", "runStarted", "runResult"].map((kind) => records.filter((x) => x.kind === kind && (x.content as { run?: string }).run === run).length), [1, 1, 1]); + const result = records.findLast((x) => x.kind === "runResult")!; + if (result.kind !== "runResult") throw new Error("缺结果"); + assert.equal(result.content.run, run); + assert.equal(result.content.process.ending, "exited"); + assert.equal(result.content.judged, null); + assert.deepEqual(runBalance(13, new Board(records).runStates()), { charged: 2, reserved: 0, balance: 11 }); + assert.equal(await s.runner.status(name), "absent"); + assert.equal(await recover(s), null); + assert.equal(s.ledger.records().length, records.length); + // 恢复后的判分：从运行目录重判，逐个实验单独判分。 + const judgedRun = await judgeDiagnosisRun(s, run, result.content.process, [{ id: "d1", metrics: [] }]); + assert.equal(judgedRun.experiments.length, 1); + const d1 = judgedRun.experiments[0]!; + assert.deepEqual([d1.id, d1.ending, d1.verdict, d1.score], ["d1", "finished", "valid", { status: "measured", value: BASELINE_SUM }]); +}); + +test("诊断运行恢复：登记了、容器不存在，确认没启动：写结果没启动、退还名额，不补启动", { timeout: 120_000 }, async (t) => { + const { s, version } = await setup(t, "notstarted", 0); + planned(s, version); + assert.equal(await recover(s), null); + const records = s.ledger.records(); + const result = records.findLast((x) => x.kind === "runResult")!; + if (result.kind !== "runResult") throw new Error("缺结果"); + assert.equal(result.content.process.ending, "notStarted"); + assert.equal(result.content.judged, null); + assert.equal(records.filter((x) => x.kind === "runStarted" && x.content.run === "r1-diagnosis-1").length, 0); + assert.deepEqual(runBalance(13, new Board(records).runStates()), { charged: 1, reserved: 0, balance: 12 }); +}); diff --git a/tests/flow-diagnosis-steps.test.ts b/tests/flow-diagnosis-steps.test.ts new file mode 100644 index 0000000..a4236f9 --- /dev/null +++ b/tests/flow-diagnosis-steps.test.ts @@ -0,0 +1,185 @@ +import assert from "node:assert/strict"; +import { appendFileSync, cpSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { test, type TestContext } from "node:test"; +import { stepList } from "../src/config/index.ts"; +import { fullConfig as checkedFull } from "../src/flow/frozen.ts"; +import { runPaths, workspaceRepo } from "../src/flow/paths.ts"; +import { groupSameExperiment, selectDiagnosisExperiments } from "../src/flow/steps/diagnosis.ts"; +import { prepareSubmission, taskArtifact, taskCode } from "../src/flow/task-code.ts"; +import type { Study } from "../src/flow/study.ts"; +import { Ledger } from "../src/ledger/index.ts"; +import { runDiagnosisRule, type DiagnosisRuleInput } from "../src/rules/index.ts"; +import type { ForwardCode } from "../src/submissions/index.ts"; +import { checkTaskFacts, diagnosisUnversionedNames, loadTask, placeBaselineArtifact } from "../src/tasks/index.ts"; +import type { JevJudgment } from "../src/vocab/index.ts"; +import { compareVersions, createRepository, saveVersion } from "../src/workspace/index.ts"; +import { fullConfig } from "./config-helpers.ts"; +import { tempDir } from "./ledger-helpers.ts"; +import { judged, r, Writer } from "./materials-helpers.ts"; + +// 诊断里程序自己做的两步（任务书 18 第 1.6 节）：判重合并的分组、选诊断实验；加上诊断实现、修复、核验的取材和存版本接线。判重合并的 Jev 问法由 jev 的测试管； +// 诊断运行和结算要真实 Docker，在 flow-diagnosis-docker.test.ts。预期值照条文手拼，输出用 rules 的函数当对照 +// （同选格测试的写法，rules 自己有独立预期值的测试）。 + +const facts = checkTaskFacts(JSON.parse(readFileSync(join(import.meta.dirname, "..", "tasks", "circle_packing", "task.json"), "utf8"))); + +const yes = (probability: number): JevJudgment => ({ question: "diagnosisDedup", probability, threshold: 0.3, yes: probability > 0.3 }) as unknown as JevJudgment; +const item = (author: string, index: number) => ({ author, index, label: `${author}.e${index + 1}` }); + +test("诊断实验判重合并：判为相同的连成一组，编号 d1… 按组里排最前的实验排，组内按作者、第几个排", () => { + const items = [item("researcher-1", 0), item("researcher-1", 1), item("researcher-2", 0), item("researcher-3", 0)]; + const pairs = [ + { a: "researcher-1.e1", b: "researcher-2.e1", judgment: yes(0.9) }, + { a: "researcher-1.e2", b: "researcher-3.e1", judgment: yes(0.1) }, + { a: "researcher-2.e1", b: "researcher-3.e1", judgment: yes(0.8) }, + ]; + // researcher-1.e1、researcher-2.e1、researcher-3.e1 连成一组（A=B，B=C）；researcher-1.e2 自成一个。 + assert.deepEqual(groupSameExperiment(items, pairs), { + experiments: [ + { experiment: "d1", sources: [{ author: "researcher-1", index: 0 }, { author: "researcher-2", index: 0 }, { author: "researcher-3", index: 0 }] }, + { experiment: "d2", sources: [{ author: "researcher-1", index: 1 }] }, + ], + }); +}); + +test("诊断实验判重合并：没有一对判相同时，每个实验各成一个", () => { + const items = [item("researcher-1", 0), item("researcher-2", 0)]; + const pairs = [{ a: "researcher-1.e1", b: "researcher-2.e1", judgment: yes(0.1) }]; + assert.deepEqual(groupSameExperiment(items, pairs).experiments.map((e) => e.experiment), ["d1", "d2"]); +}); + +/** 基线 0.96；a1、a2 各一条诊断解释，对合并后的 d1、d2 各给三个数；余额：13 次里基线用了 1 次。 */ +function study(t: TestContext, ranges: Record[]>): { s: Study; input: DiagnosisRuleInput } { + const ledger = Ledger.open(join(tempDir(t), "ledger.db")); + t.after(() => ledger.close()); + const w = new Writer(ledger); + ledger.append("code", { object: "base", version: "v-base", locations: null }); + w.run("base-1", "baseline", 0, null, judged(0.96, "main")); + w.open("r1-diagnosisProposal", "diagnosisProposal", 1, ["a1", "a2"]); + const experiment = (name: string) => ({ name, spot: `${name}的位置`, body: `${name}的做法`, metrics: [] }); + for (const [author, name] of [["a1", "实验甲"], ["a2", "实验乙"]] as const) { + w.sub("r1-diagnosisProposal", author, "diagnosisProposal", null, null, { explanations: [{ body: `${author}的解释` }], experiments: [experiment(name)] }); + } + w.close("r1-diagnosisProposal"); + w.decision("diagnosisDedup", 1, null, { + experiments: [{ experiment: "d1", sources: [{ author: "a1", index: 0 }] }, { experiment: "d2", sources: [{ author: "a2", index: 0 }] }], + }); + w.open("r1-diagnosisPrediction", "diagnosisPrediction", 1, ["a1", "a2"]); + for (const author of ["a1", "a2"]) { + const mine = ranges[author]!; + w.sub("r1-diagnosisPrediction", author, "diagnosisPrediction", null, null, { + predictions: ["d1", "d2"].map((d, i) => ({ explanation: `r1-${author}-dx1`, experiment: d, range: mine[i]! })), + body: "理由", + }); + } + w.close("r1-diagnosisPrediction"); + const config = fullConfig(); + if (config.flow.preset !== "full") throw new Error("测试配置应是完整流程"); + config.flow = { ...config.flow, diagnosis: { enabled: true, perResearcher: 2, maxExperiments: 4 }, ablation: { enabled: false } }; + const frozen = { config: checkedFull(config), steps: stepList(config), facts, prompts: null, image: "", seed: 1 }; + const input: DiagnosisRuleInput = { + baseline: 0.96, + direction: "maximize", + lambda: 0.1, + experiments: ["d1", "d2"], + explanations: ["a1", "a2"].map((author) => ({ + id: `r1-${author}-dx1`, + author, + experiments: ["d1", "d2"].map((experiment, i) => ({ experiment, range: ranges[author]![i]! })), + })), + balance: 12, + batchCap: 2, + maxExperiments: 4, + u: 0.5, + decay: 0.5, + }; + return { s: { ledger, frozen } as unknown as Study, input }; +} + +test("选诊断实验：输入照 DiagnosisRuleInput 拼（基线、每条解释对每个实验的三个数、余额从本轮之前的运行算），规则取步骤清单，种子 null", (t) => { + // 两人对 d1 预测相反（一个说涨到 1.3，一个说跌到 0.9），对 d2 都说涨不了多少。 + const { s, input } = study(t, { + a1: [r(1.2, 1.3, 1.4), r(0.95, 0.97, 0.99)], + a2: [r(0.8, 0.9, 1.0), r(0.95, 0.97, 0.99)], + }); + selectDiagnosisExperiments(s, 1); + const record = s.ledger.records().find((x) => x.kind === "decision" && x.content.decision === "selectDiagnosis")!; + const { rule: used, ...output } = runDiagnosisRule("valueMinusCost", input); + assert.deepEqual(record.content, { decision: "selectDiagnosis", round: 1, rule: "valueMinusCost", ruleVersion: used.version, params: output.params, seed: null, input, output }); + // 分歧大的实验信息价值为正：本轮做诊断，d1 在选中的组合里。 + assert.equal(output.diagnose, true); + assert.ok(output.chosen.includes("d1")); +}); + +/** 诊断实现的代码接线：真实裸仓库里存了基线；工作目录是基线副本加一处改动，再放进只读的原始产物。 */ +function codeStudy(t: TestContext): { s: Study; work: string; baseline: string } { + const dir = tempDir(t); + const ledger = Ledger.open(join(dir, "ledger.db")); + t.after(() => ledger.close()); + const task = loadTask(join(import.meta.dirname, "..", "tasks", "circle_packing")); + const repo = workspaceRepo(dir); + createRepository(repo); + const names = diagnosisUnversionedNames(task); + const saved = saveVersion(repo, task.baselineDir, names); + if (!saved.saved) throw new Error("基线存不成版本"); + const w = new Writer(ledger); + ledger.append("code", { object: "base", version: saved.version, locations: null }); + w.run("base-1", "baseline", 0, null, judged(0.96, "main")); + const baseRun = runPaths(dir, "base-1").run; + mkdirSync(baseRun, { recursive: true }); + writeFileSync(join(baseRun, "solution.json"), "{}"); + w.open("r1-diagnosisProposal", "diagnosisProposal", 1, ["a1"]); + const metric = { id: "note", meaning: "记录", unit: "无", direction: "maximize" as const, howMeasured: "数一数", missing: "recordMissing" as const }; + w.sub("r1-diagnosisProposal", "a1", "diagnosisProposal", null, null, { explanations: [{ body: "解释" }], experiments: [{ name: "实验甲", spot: "第一步", body: "换个做法", metrics: [metric] }] }); + w.close("r1-diagnosisProposal"); + w.decision("diagnosisDedup", 1, null, { experiments: [{ experiment: "d1", sources: [{ author: "a1", index: 0 }] }] }); + w.decision("selectDiagnosis", 1, null, { diagnose: true, chosen: ["d1"] }); + const work = join(dir, "work"); + cpSync(task.baselineDir, work, { recursive: true }); + appendFileSync(join(work, "initial.py"), "\n\ndef run_experiment(experiment_id, baseline_dir):\n return [], [], 0.0\n"); + placeBaselineArtifact(task, baseRun, work); + const frozen = { config: checkedFull(fullConfig()), steps: stepList(fullConfig()), facts: task.facts, prompts: null, image: "", seed: 1 }; + return { s: { dir, task, ledger, frozen } as unknown as Study, work, baseline: saved.version }; +} + +const dxTask = (kind: "diagnosisImplementation" | "diagnosisRepair" | "diagnosisVerification") => + ({ actor: "x", kind, object: "r1-diagnosis", stageId: `r1-${kind}-1` }); + +test("诊断实现的接线：存版本时跳过只读的原始产物目录；Jev 对照选中实验的原文和全名指标；工作目录放进原始产物", (t) => { + const { s, work, baseline } = codeStudy(t); + const w = new Writer(s.ledger); + w.open("r1-diagnosisImplementation-1", "diagnosisImplementation", 1, ["x"]); + const content = { locations: [{ experiment: "d1", locations: ["initial.py:90-92"] }], concerns: "无", body: "无" }; + const { known, code } = prepareSubmission(s, dxTask("diagnosisImplementation"), content, 0, work); + const saved = (known as { code: ForwardCode }).code; + if (!saved.saved) throw new Error("应存成版本"); + assert.deepEqual(saved.protectedFiles, []); + assert.ok(saved.hunks.length > 0); + assert.ok(saved.hunks.every((h) => h.path === "initial.py")); + assert.match(saved.protocol.body, /### d1 实验甲\n\n动哪一处：第一步\n\n换个做法/); + assert.deepEqual(saved.protocol.metrics.map((m) => m.id), ["d1.note"]); + assert.deepEqual(code, { object: "r1-diagnosis", version: code!.version, locations: content.locations }); + // 工作目录里放着的原始产物没有进版本：当前基线到诊断程序的改动里只有 initial.py。 + const changes = compareVersions(workspaceRepo(s.dir), baseline, code!.version); + assert.deepEqual(changes.files.map((f) => f.path), ["initial.py"]); +}); + +test("诊断实现、修复、核验取材：实现从当前基线出发，修复从上次提交的诊断程序出发，核验和修复的差异都是当前基线到诊断程序；原始产物只给实现和修复", (t) => { + const { s, work, baseline } = codeStudy(t); + const w = new Writer(s.ledger); + const content = { locations: [{ experiment: "d1", locations: ["initial.py:90-92"] }], concerns: "无", body: "无" }; + w.open("r1-diagnosisImplementation-1", "diagnosisImplementation", 1, ["x"]); + w.open("r1-diagnosisRepair-1", "diagnosisImplementation", 1, ["x"]); + w.open("r1-diagnosisVerification-1", "diagnosisVerification", 1, ["v"]); + assert.deepEqual(taskCode(s, dxTask("diagnosisImplementation")), { changes: null, startVersion: baseline }); + const { code } = prepareSubmission(s, dxTask("diagnosisImplementation"), content, 0, work); + s.ledger.append("code", code!); + const changes = compareVersions(workspaceRepo(s.dir), baseline, code!.version); + assert.deepEqual(taskCode(s, dxTask("diagnosisRepair")), { changes, startVersion: code!.version }); + assert.deepEqual(taskCode(s, dxTask("diagnosisVerification")), { changes, startVersion: null }); + const baseRun = runPaths(s.dir, "base-1").run; + assert.equal(taskArtifact(s, dxTask("diagnosisImplementation")), baseRun); + assert.equal(taskArtifact(s, dxTask("diagnosisRepair")), baseRun); + assert.equal(taskArtifact(s, dxTask("diagnosisVerification")), null); +}); diff --git a/tests/flow-next-step.test.ts b/tests/flow-next-step.test.ts index c1fb41a..88d1f8b 100644 --- a/tests/flow-next-step.test.ts +++ b/tests/flow-next-step.test.ts @@ -4,11 +4,11 @@ import { join } from "node:path"; import { test } from "node:test"; import { stepList, type ResearchConfig } from "../src/config/index.ts"; import type { AppendKind, Content, LedgerRecord } from "../src/ledger/index.ts"; -import { Board, forwardNotRunEvent, type StudyRecord } from "../src/materials/index.ts"; +import { Board, diagnosisNotRunEvent, forwardNotRunEvent, type StudyRecord } from "../src/materials/index.ts"; import { runBalance } from "../src/rules/index.ts"; import { definition, exited } from "./ledger-helpers.ts"; import { judged } from "./materials-helpers.ts"; -import { nextStep } from "../src/flow/next-step.ts"; +import { diagnosisProgram, nextStep } from "../src/flow/next-step.ts"; import type { Json, Submission, SubmissionKind } from "../src/vocab/index.ts"; import { fullConfig } from "./config-helpers.ts"; @@ -217,10 +217,30 @@ function assigned(book: Book): Book { return book.add("decision", { decision: "entries", round: 0, rule: "assignEntries", ruleVersion: 1, params: null, seed: null, input: null, output }); } -test("认领关了：分配；分配做完：开第 1 轮的提案阶段，名单是全部研究者", () => { +const DIAGNOSIS_STEPS = [ + "diagnosisProposal", "diagnosisDedupe", "diagnosisPrediction", "selectDiagnosis", + "diagnosisImplementation", "diagnosisRun", "diagnosisSettle", +]; + +/** 诊断关着（fullConfig 的写法）：第 round 轮开头的诊断七步各记一条按配置跳过（方法.md 第十一部分"步骤开关"）。 */ +function diagnosisSkipped(book: Book, round: number): Book { + for (const step of DIAGNOSIS_STEPS) { + assert.deepEqual(nextStep(book.records), { do: "skip", step: `r${round}-${step}`, reason: "配置关掉了诊断" }); + book.add("event", { event: "skippedByConfig", reason: "配置关掉了诊断", objects: [`r${round}-${step}`] }); + } + return book; +} + +/** 第零轮做完、入口分配好，第 1 轮的诊断七步也记完跳过。 */ +function inRoundOne(book: Book): Book { + return diagnosisSkipped(assigned(book), 1); +} + +test("认领关了：分配；分配做完：诊断关着，七步各记按配置跳过，再开第 1 轮的提案阶段，名单是全部研究者", () => { const book = afterRoundZero(fullConfig()); assert.deepEqual(nextStep(book.records), { do: "assignEntries" }); assigned(book); + diagnosisSkipped(book, 1); assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-proposal", stage: "proposal", round: 1, roster: R }); }); @@ -246,7 +266,7 @@ const each = (kind: SubmissionKind, objects: string[]) => R.flatMap((actor) => o /** 第 1 轮写到 last 这一步做完为止。 */ function roundOne(last: "proposal" | "dedupe" | "firstPrediction" | "critique" | "response" | "freezeProtocol" | "secondPrediction" | "selectBatch"): Book { - const book = assigned(afterRoundZero(fullConfig())); + const book = inRoundOne(afterRoundZero(fullConfig())); const order = ["proposal", "dedupe", "firstPrediction", "critique", "response", "freezeProtocol", "secondPrediction", "selectBatch"]; const steps: Record void> = { proposal: () => stage(book, "r1-proposal", "proposal", [["researcher-1", "proposal", null, P1], ["researcher-2", "proposal", null, P2], ["researcher-3", "proposal", null, P3]]), @@ -278,7 +298,7 @@ function roundOne(last: "proposal" | "dedupe" | "firstPrediction" | "critique" | } test("提案阶段一人交了两条、两人没交：只派没交的两人，提案没有对象", () => { - const book = assigned(afterRoundZero(fullConfig())).open("r1-proposal", "proposal", 1, R).sub("r1-proposal", "researcher-1", "proposal", null, P1); + const book = inRoundOne(afterRoundZero(fullConfig())).open("r1-proposal", "proposal", 1, R).sub("r1-proposal", "researcher-1", "proposal", null, P1); book.sub("r1-proposal", "researcher-1", "proposal", null, "r1-researcher-1-2"); const tasks = ["researcher-2", "researcher-3"].map((actor) => ({ actor, kind: "proposal", object: null })); assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-proposal", tasks }); @@ -350,7 +370,7 @@ test("关掉第一次预测：第 1 轮写一条按配置跳过（对象 r1-firs const config = fullConfig(); if (config.flow.preset !== "full") throw new Error("测试配置应是完整流程"); config.flow.firstPrediction = false; - const book = assigned(afterRoundZero(config)); + const book = inRoundOne(afterRoundZero(config)); stage(book, "r1-proposal", "proposal", [["researcher-1", "proposal", null, P1], ["researcher-2", "proposal", null, P2], ["researcher-3", "proposal", null, P3]]); const output = { fresh: [{ candidate: "r1-c1", proposals: [P1] }, { candidate: "r1-c2", proposals: [P2, P3] }], carried: [] }; book.add("decision", { decision: "dedup", round: 1, rule: "jevSameChange", ruleVersion: 1, params: null, seed: null, input: null, output }); @@ -648,7 +668,8 @@ test("结算和押的因子写完：开收尾；收尾关闭后解封本轮选 book.add("unseal", { candidate: "r1-c1", seals: [1] }); assert.deepEqual(nextStep(book.records), { do: "unsealChosen", round: 1 }); book.add("unseal", { candidate: "r1-c2", seals: [2] }); - // 余额还有（已扣 5 次）：开第 2 轮。 + // 余额还有（已扣 5 次）：开第 2 轮，先是诊断七步的按配置跳过。 + diagnosisSkipped(book, 2); assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r2-proposal", stage: "proposal", round: 2, roster: R }); }); @@ -679,7 +700,7 @@ test("余额为 0：不开下一轮，走到导出停下，原因写明", () => assert.deepEqual(nextStep(book.records), { do: "halt", reason: "导出还没写", objects: [] }); }); -test("配置关掉消融、又采纳了多因子改法：先记按配置跳过，再在解释停下", () => { +test("配置关掉消融、又采纳了多因子改法：⑬–⑰ 整段（含解释）各记按配置跳过，再开收尾", () => { const book = firstVerification(batchTwo(1), true, true); for (const c of C) recordForward(book, c, 1, "valid"); book.add("decision", { decision: "adopt", round: 1, rule: "adopt", ruleVersion: 1, params: null, seed: null, input: null, output: { adopted: ADOPTED } }); @@ -693,9 +714,192 @@ test("配置关掉消融、又采纳了多因子改法：先记按配置跳过 study.config = config as unknown as Json; study.steps = stepList(config) as unknown as Json; const reason = "配置关掉了消融"; - assert.deepEqual(nextStep(book.records), { do: "skip", step: "r1-listCells", reason }); - book.add("event", { event: "skippedByConfig", reason, objects: ["r1-listCells"] }); - const next = nextStep(book.records); - assert.equal(next.do, "halt"); - assert.match((next as { reason: string }).reason, /不交三个数的解释还没写/); + // 方法.md 4.3 ⑫：解释是 ⑭，和列格、选格、消融、结算一起不做；没有"解释停下"的情形。 + for (const step of ["listCells", "explanation", "selectCells", "ablationImplementation", "ablationVerification", "ablationRun", "settle"]) { + assert.deepEqual(nextStep(book.records), { do: "skip", step: `r1-${step}`, reason }); + book.add("event", { event: "skippedByConfig", reason, objects: [`r1-${step}`] }); + } + assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-wrapUp", stage: "wrapUp", round: 1, roster: R }); +}); + +// --------------------------------------------------------------------------- +// 诊断（任务书 18 第 1.6 节，方法.md 4.3 诊 1–诊 4）：预期动作手写，不调模型、Jev 或 Docker。 +// --------------------------------------------------------------------------- + +const DX = "r1-diagnosis"; +const DX_ACTOR = "experimenter-r1-diagnosis"; +const DX_VERIFIER = "verifier-r1-diagnosis"; + +/** 诊断开着的配置：A = 2、D = 4（方法.md 6.11 现行值），消融关；maxRepairs 和运行次数可调。 */ +function diagnosisConfig(maxRepairs: number, runs: number): ResearchConfig { + const config = { ...fullConfig(), maxRepairs, runs }; + if (config.flow.preset !== "full") throw new Error("测试配置应是完整流程"); + config.flow = { ...config.flow, diagnosis: { enabled: true, perResearcher: 2, maxExperiments: 4 }, ablation: { enabled: false } }; + return config; +} + +/** 第零轮做完、入口分配好；诊断开着，第 1 轮的诊断从 r1-diagnosisProposal 开始。 */ +function diagnosisRoundOne(maxRepairs: number, runs: number): Book { + return assigned(afterRoundZero(diagnosisConfig(maxRepairs, runs))); +} + +const dxTasks = (kind: SubmissionKind) => R.map((actor) => ({ actor, kind, object: null })); + +/** 诊 1、诊 2 阶段关闭，两个实验合并成 d1、d2，诊 3 的决定写下。 */ +function selected(book: Book, diagnose: boolean, chosen: string[]): Book { + for (const [stageId, stage, kind] of [["r1-diagnosisProposal", "diagnosisProposal", "diagnosisProposal"], ["r1-diagnosisPrediction", "diagnosisPrediction", "diagnosisPrediction"]] as const) { + book.open(stageId, stage, 1, R); + for (const r of R) book.sub(stageId, r, kind, null, null); + book.add("stageClose", { stageId }); + if (kind === "diagnosisProposal") { + book.add("decision", { decision: "diagnosisDedup", round: 1, rule: "jevSameExperiment", ruleVersion: 1, params: null, seed: null, input: null, output: { experiments: [] } }); + } + } + return book.add("decision", { decision: "selectDiagnosis", round: 1, rule: "valueMinusCost", ruleVersion: 1, params: null, seed: null, input: null, output: { diagnose, chosen } }); +} + +/** 诊断实现（n = 1）或修复（n ≥ 2）阶段：实验员交一份，关闭。 */ +function dxWork(book: Book, n: number): Book { + const stageId = n === 1 ? "r1-diagnosisImplementation-1" : `r1-diagnosisRepair-${n - 1}`; + book.open(stageId, "diagnosisImplementation", 1, [DX_ACTOR]); + book.sub(stageId, DX_ACTOR, n === 1 ? "diagnosisImplementation" : "diagnosisRepair", DX, null); + return book.add("stageClose", { stageId }); +} + +/** 第 n 次诊断核验：results 是每个选中实验"是否通过"。 */ +function dxVerified(book: Book, n: number, results: Record): Book { + const stageId = `r1-diagnosisVerification-${n}`; + book.open(stageId, "diagnosisVerification", 1, [DX_VERIFIER]); + const content: Submission<"diagnosisVerification"> = { + experiments: Object.entries(results).map(([experiment, ok]) => ({ experiment, followed: ok, nothingElse: true, where: ok ? "initial.py:10" : "没找到" })), + body: "无", + }; + book.add("submission", { stageId, author: DX_VERIFIER, kind: "diagnosisVerification", target: DX, item: null, version: 1, content, jev: null }); + return book.add("stageClose", { stageId }); +} + +/** 一次诊断运行：没启动，或启动了（运行记录不带判分）。 */ +function dxRun(book: Book, n: number, ending: "valid" | "notStarted"): Book { + const run = `r1-diagnosis-${n}`; + book.add("runPlanned", { run, purpose: "diagnosis", round: 1, object: DX, definition }); + if (ending === "notStarted") return book.add("runResult", { run, process: { ending, reason: "Docker 未启动" }, judged: null }); + book.add("runStarted", { run }); + return book.add("runResult", { run, process: exited, judged: null }); +} + +const openProposal = { do: "openStage", stageId: "r1-proposal", stage: "proposal", round: 1, roster: R }; + +test("诊断开着：诊 1 开阶段，每人一个任务、没有对象；交齐关闭后判重合并，再开诊断预测，同样每人一个任务", () => { + const book = diagnosisRoundOne(1, 13); + assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-diagnosisProposal", stage: "diagnosisProposal", round: 1, roster: R }); + book.open("r1-diagnosisProposal", "diagnosisProposal", 1, R); + assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-diagnosisProposal", tasks: dxTasks("diagnosisProposal") }); + for (const r of R) book.sub("r1-diagnosisProposal", r, "diagnosisProposal", null, null); + assert.deepEqual(nextStep(book.records), { do: "closeStage", stageId: "r1-diagnosisProposal" }); + book.add("stageClose", { stageId: "r1-diagnosisProposal" }); + assert.deepEqual(nextStep(book.records), { do: "diagnosisDedupe", round: 1 }); + book.add("decision", { decision: "diagnosisDedup", round: 1, rule: "jevSameExperiment", ruleVersion: 1, params: null, seed: null, input: null, output: { experiments: [] } }); + assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-diagnosisPrediction", stage: "diagnosisPrediction", round: 1, roster: R }); + book.open("r1-diagnosisPrediction", "diagnosisPrediction", 1, R); + assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-diagnosisPrediction", tasks: dxTasks("diagnosisPrediction") }); + for (const r of R) book.sub("r1-diagnosisPrediction", r, "diagnosisPrediction", null, null); + book.add("stageClose", { stageId: "r1-diagnosisPrediction" }); + assert.deepEqual(nextStep(book.records), { do: "selectDiagnosis", round: 1 }); +}); + +test("余额不到 2 次：诊 1–诊 4 都不做，不开阶段、不写记录，直接开提案（运行 2 次，基线用掉 1 次，余额 1）", () => { + const book = diagnosisRoundOne(1, 2); + assert.deepEqual(nextStep(book.records), openProposal); +}); + +test("诊 3 定了不诊断：诊 4 的几步都不做，直接开提案；决定记录照写", () => { + const book = selected(diagnosisRoundOne(1, 13), false, []); + assert.deepEqual(nextStep(book.records), openProposal); +}); + +test("诊断实现与核验：一位实验员对本轮诊断，一位核验员逐个实验交结果；全通过就运行，没启动起新编号，已登记没结果要先恢复", () => { + const book = selected(diagnosisRoundOne(1, 13), true, ["d1", "d2"]); + assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-diagnosisImplementation-1", stage: "diagnosisImplementation", round: 1, roster: [DX_ACTOR] }); + book.open("r1-diagnosisImplementation-1", "diagnosisImplementation", 1, [DX_ACTOR]); + assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-diagnosisImplementation-1", tasks: [{ actor: DX_ACTOR, kind: "diagnosisImplementation", object: DX }] }); + book.sub("r1-diagnosisImplementation-1", DX_ACTOR, "diagnosisImplementation", DX, null); + book.add("stageClose", { stageId: "r1-diagnosisImplementation-1" }); + assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-diagnosisVerification-1", stage: "diagnosisVerification", round: 1, roster: [DX_VERIFIER] }); + dxVerified(book, 1, { d1: true, d2: true }); + assert.deepEqual(nextStep(book.records), { do: "diagnosisRun", round: 1, run: "r1-diagnosis-1" }); + dxRun(book, 1, "notStarted"); + assert.deepEqual(nextStep(book.records), { do: "diagnosisRun", round: 1, run: "r1-diagnosis-2" }); + assert.deepEqual(runBalance(13, new Board(book.records).runStates()), { charged: 1, reserved: 0, balance: 12 }); + book.add("runPlanned", { run: "r1-diagnosis-2", purpose: "diagnosis", round: 1, object: DX, definition }); + assert.throws(() => nextStep(book.records), /诊断运行 r1-diagnosis-2 登记了还没有结果：恢复入口应当已经核对过/); + book.add("runStarted", { run: "r1-diagnosis-2" }); + book.add("runResult", { run: "r1-diagnosis-2", process: exited, judged: null }); + assert.deepEqual(nextStep(book.records), { do: "diagnosisSettle", round: 1 }); + // 没启动的那次退还名额：基线 1 次、诊断 1 次，诊断运行只扣一次。 + assert.deepEqual(runBalance(13, new Board(book.records).runStates()), { charged: 2, reserved: 0, balance: 11 }); + book.add("diagnosisSettlement", { round: 1, result: null }); + assert.deepEqual(nextStep(book.records), openProposal); +}); + +test("核验有实验不通过：只退回修复，只重验一次；用完修复仍不通过的实验拿掉，跑的只是剩下的实验", () => { + const book = selected(diagnosisRoundOne(1, 13), true, ["d1", "d2"]); + dxWork(book, 1); + dxVerified(book, 1, { d1: true, d2: false }); + assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-diagnosisRepair-1", stage: "diagnosisImplementation", round: 1, roster: [DX_ACTOR] }); + book.open("r1-diagnosisRepair-1", "diagnosisImplementation", 1, [DX_ACTOR]); + assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-diagnosisRepair-1", tasks: [{ actor: DX_ACTOR, kind: "diagnosisRepair", object: DX }] }); + book.sub("r1-diagnosisRepair-1", DX_ACTOR, "diagnosisRepair", DX, null); + book.add("stageClose", { stageId: "r1-diagnosisRepair-1" }); + assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-diagnosisVerification-2", stage: "diagnosisVerification", round: 1, roster: [DX_VERIFIER] }); + dxVerified(book, 2, { d1: true, d2: false }); + // 修复次数上限 1 已用完：d2 拿掉，d1 照跑；没有"不跑"事件。 + assert.deepEqual(nextStep(book.records), { do: "diagnosisRun", round: 1, run: "r1-diagnosis-1" }); + const dropped = diagnosisProgram(new Board(book.records), 1, 1); + assert.deepEqual(dropped.experiments, ["d1"]); + assert.deepEqual(dropped.dropped.map((d) => d.experiment), ["d2"]); + assert.match(dropped.dropped[0]!.reason, /第 2 次核验仍不通过，修复次数上限 1 已用完：没照写法做；没找到/); +}); + +test("修复后上次通过的实验又不通过：以最近一次核验为准，修复前通过过也不能跑", () => { + const book = selected(diagnosisRoundOne(1, 13), true, ["d1", "d2"]); + dxWork(book, 1); + dxVerified(book, 1, { d1: true, d2: false }); + dxWork(book, 2); + dxVerified(book, 2, { d1: false, d2: true }); + assert.deepEqual(diagnosisProgram(new Board(book.records), 1, 1).experiments, ["d2"]); + assert.deepEqual(nextStep(book.records), { do: "diagnosisRun", round: 1, run: "r1-diagnosis-1" }); +}); + +test("maxRepairs = 0：第一次核验不通过的实验就拿掉，不开修复阶段", () => { + const book = selected(diagnosisRoundOne(0, 13), true, ["d1", "d2"]); + dxWork(book, 1); + dxVerified(book, 1, { d1: false, d2: true }); + assert.deepEqual(nextStep(book.records), { do: "diagnosisRun", round: 1, run: "r1-diagnosis-1" }); + assert.deepEqual(diagnosisProgram(new Board(book.records), 1, 0).experiments, ["d2"]); +}); + +test("选中的实验一个都不剩：写不跑事件（一次），不登记运行、不扣次，不结算，直接开提案", () => { + const book = selected(diagnosisRoundOne(1, 13), true, ["d1", "d2"]); + dxWork(book, 1); + dxVerified(book, 1, { d1: false, d2: false }); + dxWork(book, 2); + dxVerified(book, 2, { d1: false, d2: false }); + const next = nextStep(book.records); + assert.equal(next.do, "diagnosisNotRun"); + if (next.do !== "diagnosisNotRun") throw new Error("应写不跑事件"); + assert.equal(next.round, 1); + assert.match(next.reason, /选中的实验第 2 次核验都不通过，修复次数上限 1 已用完：d1：.*；d2：/); + book.add("event", diagnosisNotRunEvent(1, next.reason)); + assert.deepEqual(nextStep(book.records), openProposal); + assert.deepEqual(runBalance(13, new Board(book.records).runStates()), { charged: 1, reserved: 0, balance: 12 }); +}); + +test("诊断运行扣次以后：余额只算本轮开始之前的运行，诊断这一段的后几步不会算成不诊断", () => { + // 运行 3 次：本轮开始前余额 2，诊断运行扣 1 次后余额 1。诊断这一段的后几步照常走，不会因余额变小算成"不诊断"。 + const book = selected(diagnosisRoundOne(1, 3), true, ["d1"]); + dxWork(book, 1); + dxVerified(book, 1, { d1: true }); + dxRun(book, 1, "valid"); + assert.equal(runBalance(3, new Board(book.records).runStates()).balance, 1); + assert.deepEqual(nextStep(book.records), { do: "diagnosisSettle", round: 1 }); }); diff --git a/tests/jev.test.ts b/tests/jev.test.ts index bf393b8..2db125a 100644 --- a/tests/jev.test.ts +++ b/tests/jev.test.ts @@ -3,7 +3,17 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { test, type TestContext } from "node:test"; -import { checkHunk, checkProtocol, codeHunks, judgeSameChange, type FrozenProtocol, type ProtocolRules } from "../src/jev/index.ts"; +import { + checkExperiment, + checkHunk, + checkProtocol, + codeHunks, + judgeSameChange, + judgeSameExperiment, + type DiagnosisExperiment, + type FrozenProtocol, + type ProtocolRules, +} from "../src/jev/index.ts"; import { hash, type Json } from "../src/vocab/index.ts"; import { compareVersions, createRepository, saveVersion } from "../src/workspace/index.ts"; @@ -307,3 +317,51 @@ test("把 workspace 的比较结果切成差异块，每块从 @@ 行起", (t) = assert.equal(hunks[0]?.text, "@@ -1,5 +1,5 @@\n line 1\n-line 2\n+changed 2\n line 3\n line 4\n line 5"); assert.equal(hunks[2]?.text, "@@ -0,0 +1 @@\n+x = 1"); }); + +// --------------------------------------------------------------------------- +// 诊断实验的判重和冻结检查（方法.md 4.3 诊 1、诊 2） +// --------------------------------------------------------------------------- + +const EXPERIMENT: DiagnosisExperiment = { + name: "拿掉收尾", + spot: "solve.py 的收尾修复", + body: "拿掉收尾修复，直接交出原始答案", + metrics: [{ id: "raw-score", meaning: "收尾之前的分数", unit: "无", direction: "maximize", howMeasured: "收尾之前的分数", missing: "recordMissing" }], +}; + +test("诊断实验判重：一对一次请求，只给名称、动哪一处、怎么动；阈值暂用判重的 0.3", async (t) => { + const sent = fakeJev(t, answerAll(() => 0.31)); + const other = { name: "强起点", spot: "solve.py 的起点", body: "换成更强的起点" }; + const j = await judgeSameExperiment("jev-x", EXPERIMENT, other); + assert.equal(sent.length, 1); + assert.deepEqual(Object.keys(sent[0]!.body.questions), ["same_experiment"]); + assert.deepEqual(sent[0]!.body.state.experiment_a, { name: "拿掉收尾", spot: "solve.py 的收尾修复", body: "拿掉收尾修复，直接交出原始答案" }); + assert.deepEqual(sent[0]!.body.state.experiment_b, other); + assert.deepEqual([j.question, j.threshold, j.yes, j.requestId], ["same_experiment", 0.3, true, "req_1"]); +}); + +test("诊断实验判重：概率等于阈值判否", async (t) => { + fakeJev(t, answerAll(() => 0.3)); + assert.equal((await judgeSameExperiment("jev-x", EXPERIMENT, EXPERIMENT)).yes, false); +}); + +test("诊断实验冻结检查：三问各一次请求，材料相同，问法是诊断的变体；有一问判是就标出", async (t) => { + const sent = fakeJev(t, answerAll((name) => (name === "changes_measurement" ? 0.22 : 0.1))); + const result = await checkExperiment("jev-x", EXPERIMENT); + assert.deepEqual( + result.judgments.map((j) => [j.question, j.threshold, j.yes]), + [["self_contradictory", 0.5, false], ["changes_measurement", 0.21, true], ["too_vague", 0.78, false]], + ); + assert.equal(result.flagged, true); + assert.deepEqual(sent.map((s) => Object.keys(s.body.questions)), [["self_contradictory"], ["changes_measurement"], ["too_vague"]]); + for (const s of sent) assert.deepEqual(s.body.state, sent[0]!.body.state); + assert.deepEqual(sent[0]!.body.state.experiment, { + name: "拿掉收尾", + spot: "solve.py 的收尾修复", + body: "拿掉收尾修复，直接交出原始答案", + metrics: [{ id: "raw-score", howMeasured: "收尾之前的分数", missing: "recordMissing" }], + }); + // 材料说明和问法都是诊断的变体，哈希和冻结协议同名三问的不同。 + const protocol = await checkProtocol("jev-x", PROTOCOL); + result.judgments.forEach((j, i) => assert.notEqual(j.questionHash, protocol.judgments[i]!.questionHash, j.question)); +}); diff --git a/tests/ledger-helpers.ts b/tests/ledger-helpers.ts index 5598a6f..f0f5e9f 100644 --- a/tests/ledger-helpers.ts +++ b/tests/ledger-helpers.ts @@ -80,7 +80,7 @@ export function closeStage(ledger: Ledger, stageId: string): number { } /** 登记一次运行。 */ -export function planRun(ledger: Ledger, run: string, purpose: "baseline" | "forward" | "ablation", round: number): number { +export function planRun(ledger: Ledger, run: string, purpose: "baseline" | "forward" | "ablation" | "diagnosis", round: number): number { return ledger.append("runPlanned", { run, purpose, round, object: purpose === "baseline" ? null : `obj-${run}`, definition }).seq; } diff --git a/tests/ledger-rejections.test.ts b/tests/ledger-rejections.test.ts index fd58c17..7611acd 100644 --- a/tests/ledger-rejections.test.ts +++ b/tests/ledger-rejections.test.ts @@ -16,12 +16,13 @@ import { wrapUp, } from "./ledger-helpers.ts"; -// 写入时由 ledger 拒绝的四种情况（docs/结构.md 第四部分"写入时由 ledger 拒绝的几种情况"），每种一条拒收、一条对应的正常情况。 +// 写入时由 ledger 拒绝的五种情况（docs/结构.md 第四部分"写入时由 ledger 拒绝的几种情况"），每种一条拒收、一条对应的正常情况。 // 预期值来自文字： // 1. 只收开着的阶段（有开阶段记录、没有关阶段记录）的提交，别的拒收； // 2. 要替换的那一份不是本人在这个阶段、对同一个对象的最新一版 → 拒收；实现、修复、核验、消融实现、消融核验不能替换（方法.md 第五部分）； // 3. 运行结果写在"确认启动"之前 → 拒收；"没启动"的结果不需要启动记录（方法.md 第七部分）； -// 4. 这一轮消融已经有格确认启动了，才来关这一轮的解释阶段 → 拒收（方法.md 4.3 ⑭）。 +// 4. 这一轮消融已经有格确认启动了，才来关这一轮的解释阶段 → 拒收（方法.md 4.3 ⑭）； +// 5. 这一轮的诊断运行已经确认启动了，才来关这一轮的诊断预测阶段 → 拒收（方法.md 4.3 诊 2）；诊断实现、修复、核验和实现、核验一样不能替换。 function submitWrapUp(ledger: Ledger, stageId: string, author: string, version: number, body = "收尾") { return ledger.append("submission", { stageId, author, target: null, item: null, version, jev: null, kind: "wrapUp", content: wrapUp(body) }); @@ -156,3 +157,53 @@ test("拒收 4 的正常情况：只登记没启动、别的轮的消融启动 closeStage(ledger, "wrap"); assert.equal(ledger.stageClosed("wrap"), true); }); + +test("拒收 5：这一轮的诊断运行已经确认启动，就不能关这一轮的诊断预测阶段", (t) => { + const { ledger } = freshLedger(t); + openStage(ledger, "dpred", "diagnosisPrediction", 2); + planRun(ledger, "diagrun", "diagnosis", 2); + ledger.append("runStarted", { run: "diagrun" }); + assert.throws(() => closeStage(ledger, "dpred"), /诊断预测冻结必须早于诊断运行启动/); + assert.equal(ledger.stageClosed("dpred"), false); +}); + +test("拒收 5 的正常情况：只登记没启动、别的轮的诊断启动、本轮消融或正向启动，都不挡关诊断预测阶段；消融启动也不被诊断运行连累", (t) => { + const { ledger } = freshLedger(t); + openStage(ledger, "dpred", "diagnosisPrediction", 2); + planRun(ledger, "diag-planned", "diagnosis", 2); // 只登记，还没确认启动 + planRun(ledger, "diag-round1", "diagnosis", 1); + ledger.append("runStarted", { run: "diag-round1" }); + planRun(ledger, "fwd-round2", "forward", 2); + ledger.append("runStarted", { run: "fwd-round2" }); + closeStage(ledger, "dpred"); + assert.equal(ledger.stageClosed("dpred"), true); + // 诊断运行启动后，解释阶段照旧关得掉：两种用途各管各的。 + openStage(ledger, "expl", "explanation", 2); + planRun(ledger, "diag-round2", "diagnosis", 2); + ledger.append("runStarted", { run: "diag-round2" }); + closeStage(ledger, "expl"); + assert.equal(ledger.stageClosed("expl"), true); +}); + +test("拒收 2：诊断实现、诊断修复、诊断核验只收第一版，不能替换；诊断预测可以替换", (t) => { + const { ledger } = freshLedger(t); + openStage(ledger, "dimpl", "diagnosisImplementation", 1, ["e1"]); + openStage(ledger, "dver", "diagnosisVerification", 1, ["v1"]); + openStage(ledger, "dpred", "diagnosisPrediction", 1); + const base = { author: "e1", target: "r1-diagnosis", item: null }; + const code = { locations: [{ experiment: "d1", locations: ["solve.py:3-9"] }], concerns: "无", body: "无" }; + const cases = [ + { ...base, stageId: "dimpl", kind: "diagnosisImplementation" as const, content: code }, + { ...base, stageId: "dimpl", author: "e2", kind: "diagnosisRepair" as const, content: code }, + { ...base, author: "v1", stageId: "dver", kind: "diagnosisVerification" as const, content: { experiments: [], body: "无" } }, + ]; + for (const c of cases) { + ledger.append("submission", { ...c, version: 1, jev: null }); + assert.throws(() => ledger.append("submission", { ...c, version: 2, jev: null }), /不能替换/, c.kind); + } + const slot = { stageId: "dpred", author: "r1", target: null, item: null, kind: "diagnosisPrediction" as const }; + const content = { predictions: [{ explanation: "r1-r1-dx1", experiment: "d1", range: range(1, 2, 3) }], body: "理由" }; + ledger.append("submission", { ...slot, version: 1, jev: null, content }); + ledger.append("submission", { ...slot, version: 2, jev: null, content }); + assert.equal(ledger.latestSubmission(slot)?.content.version, 2); +}); diff --git a/tests/ledger-store.test.ts b/tests/ledger-store.test.ts index 9ef4a56..d72c2a6 100644 --- a/tests/ledger-store.test.ts +++ b/tests/ledger-store.test.ts @@ -438,6 +438,16 @@ test("关掉再打开：读出的记录不变，序号接着往下排", (t) => { assert.equal(reopened.append("runStarted", { run: "run1" }).seq, 9); }); +test("唯一约束：诊断结算按轮次，同一轮两次报错；同一轮的消融结算另算，不受它影响", (t) => { + const { ledger, path } = freshLedger(t); + ledger.append("diagnosisSettlement", { round: 1, result: { experiments: [] } }); + assert.throws(() => ledger.append("diagnosisSettlement", { round: 1, result: { experiments: [] } }), /diagnosis_settlement_once/); + ledger.append("diagnosisSettlement", { round: 2, result: { experiments: [] } }); + ledger.append("settlement", { round: 1, result: { cells: {} } }); + assert.throws(() => ledger.append("diagnosisSettlement", { round: 3, run: "x", result: {} } as never), /diagnosisSettlement/); + assert.equal(countRows(path), 3); +}); + test("打开：表结构版本不对报错；不是空文件也不是账本的库报错", (t) => { const dir = tempDir(t); const wrongVersion = join(dir, "v99.db"); @@ -446,12 +456,12 @@ test("打开：表结构版本不对报错；不是空文件也不是账本的 db.close(); assert.throws(() => Ledger.open(wrongVersion), /版本是 99/); - // 版本 3 是退回还没有对象字段时的账本，不迁移，打开报错。 - const previous = join(dir, "v3.db"); - const v3 = new DatabaseSync(previous); - v3.exec("PRAGMA user_version = 3"); - v3.close(); - assert.throws(() => Ledger.open(previous), /版本是 3.*只认版本 4/s); + // 版本 4 是还没有诊断结算和诊断运行用途时的账本，不迁移，打开报错。 + const previous = join(dir, "v4.db"); + const v4 = new DatabaseSync(previous); + v4.exec("PRAGMA user_version = 4"); + v4.close(); + assert.throws(() => Ledger.open(previous), /版本是 4.*只认版本 5/s); const foreign = join(dir, "foreign.db"); const other = new DatabaseSync(foreign); diff --git a/tests/materials-diagnosis.test.ts b/tests/materials-diagnosis.test.ts new file mode 100644 index 0000000..d527b49 --- /dev/null +++ b/tests/materials-diagnosis.test.ts @@ -0,0 +1,337 @@ +import assert from "node:assert/strict"; +import { test, type TestContext } from "node:test"; +import { Ledger } from "../src/ledger/index.ts"; +import { + Board, + buildMaterial, + type CodeChanges, + diagnosisExplanationId, + diagnosisId, + diagnosisNotRunEvent, + type DiagnosisSettlement, + type StudyFacts, + type TaskMaterial, +} from "../src/materials/index.ts"; +import { selectDiagnosis, settleDiagnosis, type DiagnosisRuleInput, type SettleDiagnosisInput } from "../src/rules/index.ts"; +import type { SubmissionKind } from "../src/vocab/index.ts"; +import { freshLedger } from "./ledger-helpers.ts"; +import { batch, CHANGES_C1, everything, judged, material, NONE, P_A1, prompts, protocolOf, r, study, TABLE_C1, Writer } from "./materials-helpers.ts"; + +// 诊断的材料（任务书 18 第 1.5 节、方法.md 第三部分、4.3 诊 1–诊 4）：在真实账本里写一轮诊断，看每个任务该有的有、不该有的没有。 +// 研究者 a1、a2，实验员 x1，核验员 v1。基线 0.96。三个合并后的诊断实验：d1（a1、a2 的"强起点"合并）、d2（a1 的）、d3（a2 的第二个）。 +// a1 的诊断预测数 7.771…、a2 的 8.881…是好认的数：任何别人的材料里都不该出现。 +// 选实验的决定、结算都用 rules 的函数算（不手写数）；这里只查材料怎么读它们。 + +const SEALED = { a1: ["7.771", "7.772", "7.773"], a2: ["8.881", "8.882", "8.883"] }; +const D = diagnosisId(1); +const headings = (m: TaskMaterial) => m.sections.map((s) => s.heading); +const has = (m: TaskMaterial, text: string) => assert.ok(everything(m).includes(text), `材料里应有"${text}"`); +const lacks = (m: TaskMaterial, text: string) => assert.ok(!everything(m).includes(text), `材料里不该有"${text}"`); + +const experiment = (name: string, spot: string) => ({ name, spot, body: `${name}：${spot}上怎么动`, metrics: [] }); +const gapExperiment = { ...experiment("拿掉收尾", "solve.py 收尾"), metrics: [{ id: "gap", meaning: "间隙", unit: "无", direction: "minimize" as const, howMeasured: "最后量一次", missing: "recordMissing" as const }] }; + +/** 两条解释对三个实验的三个数：a1 说 d1 能涨很多，a2 说 d1 涨不了；d2、d3 两人差不多。 */ +const ranges = { + [diagnosisExplanationId(1, "a1", 1)]: { author: "a1", d1: r(1.1, 1.25, 1.3), d2: r(0.95, 1.0, 1.05), d3: r(0.95, 1.0, 1.05) }, + [diagnosisExplanationId(1, "a2", 1)]: { author: "a2", d1: r(0.9, 0.95, 0.99), d2: r(0.95, 1.0, 1.05), d3: r(0.95, 1.0, 1.05) }, +}; +const ruleInput = (balance: number, same: boolean): DiagnosisRuleInput => ({ + baseline: 0.96, + direction: "maximize", + lambda: 0.1, + experiments: ["d1", "d2", "d3"], + explanations: Object.entries(ranges).map(([id, x]) => ({ + id, + author: x.author, + experiments: (["d1", "d2", "d3"] as const).map((e) => ({ experiment: e, range: same ? ranges[diagnosisExplanationId(1, "a1", 1)]![e] : x[e] })), + })), + balance, + batchCap: 1, + u: 0.5, + decay: 0.5, + maxExperiments: 2, +}); + +type Step = "diagProposalOpen" | "diagProposalClosed" | "diagPredictionOpen" | "diagPredictionClosed" | "selected" | "implemented" | "verifiedFailed" | "repairOpen" | "run" | "settled"; +const ORDER: Step[] = ["diagProposalOpen", "diagProposalClosed", "diagPredictionOpen", "diagPredictionClosed", "selected", "implemented", "verifiedFailed", "repairOpen", "run", "settled"]; + +/** 第 1 轮诊断写到某一步（含）为止。选中的是净值最大的两个实验；d 里验证不通过的那个在结算里记证据不足。 */ +function diagnosisStudy(t: TestContext, upTo: Step): { l: Ledger; w: Writer; chosen: string[] } { + const { ledger: l } = freshLedger(t); + const w = new Writer(l); + const reached = (s: Step) => ORDER.indexOf(upTo) >= ORDER.indexOf(s); + l.append("code", { object: "base", version: "v-base", locations: null }); + w.run("run-0", "baseline", 0, null, judged(0.96, "main")); + w.open("s1-dp", "diagnosisProposal", 1, ["a1", "a2"]); + w.sub("s1-dp", "a1", "diagnosisProposal", null, null, { + explanations: [{ body: "甲的诊断解释：卡在起点 UNIQUE-A1" }], + experiments: [experiment("强起点", "solve.py 起点"), gapExperiment], + }); + if (!reached("diagProposalClosed")) return { l, w, chosen: [] }; + w.sub("s1-dp", "a2", "diagnosisProposal", null, null, { + explanations: [{ body: "乙的诊断解释：卡在收尾 UNIQUE-A2" }], + experiments: [experiment("强起点", "solve.py 起点（乙的写法）"), experiment("放宽间隙", "solve.py 间隙限制")], + }); + w.close("s1-dp"); + w.decision("diagnosisDedup", 1, null, { + experiments: [ + { experiment: "d1", sources: [{ author: "a1", index: 0 }, { author: "a2", index: 0 }] }, + { experiment: "d2", sources: [{ author: "a1", index: 1 }] }, + { experiment: "d3", sources: [{ author: "a2", index: 1 }] }, + ], + }); + w.open("s1-dpred", "diagnosisPrediction", 1, ["a1", "a2"]); + const pred = (author: "a1" | "a2", numbers: string[]) => + (["d1", "d2", "d3"] as const).map((e) => ({ explanation: diagnosisExplanationId(1, author, 1), experiment: e, range: e === "d1" ? r(...(numbers.map(Number) as [number, number, number])) : ranges[diagnosisExplanationId(1, author, 1)]![e] })); + w.sub("s1-dpred", "a1", "diagnosisPrediction", null, null, { predictions: pred("a1", SEALED.a1), body: "甲的预测理由" }); + if (!reached("diagPredictionClosed")) return { l, w, chosen: [] }; + w.sub("s1-dpred", "a2", "diagnosisPrediction", null, null, { predictions: pred("a2", SEALED.a2), body: "乙的预测理由" }); + w.close("s1-dpred"); + const input = ruleInput(9, false); + const decision = selectDiagnosis(input); + assert.equal(decision.diagnose, true); + const chosen = decision.chosen; + assert.equal(chosen.length, 2); + if (!reached("selected")) return { l, w, chosen }; + w.decision("selectDiagnosis", 1, input, decision); + w.open("s1-dimpl", "diagnosisImplementation", 1, ["x1"]); + const table = chosen.map((experiment, i) => ({ experiment, locations: [`solve.py:${10 + 10 * i}-${19 + 10 * i}`] })); + w.sub("s1-dimpl", "x1", "diagnosisImplementation", D, null, { locations: table, concerns: "收尾里有个旧开关", body: "诊断实现说明 UNIQUE-X1" }); + w.close("s1-dimpl"); + l.append("code", { object: D, version: "v-diag", locations: table as never }); + if (!reached("implemented")) return { l, w, chosen }; + w.open("s1-dver", "diagnosisVerification", 1, ["v1"]); + w.sub("s1-dver", "v1", "diagnosisVerification", D, null, { + experiments: chosen.map((experiment, i) => ({ experiment, followed: i === 0, nothingElse: true, where: `solve.py:${10 + 10 * i}` })), + body: "核验员：第二个实验没照写法做", + }); + w.close("s1-dver"); + if (!reached("repairOpen")) return { l, w, chosen }; + w.open("s1-dimpl2", "diagnosisImplementation", 1, ["x1"]); + if (!reached("run")) return { l, w, chosen }; + w.run("run-1", "diagnosis", 1, D, null); // 诊断运行的记录里没有判分：逐个实验的结果在结算里 + if (!reached("settled")) return { l, w, chosen }; + const settleInput: SettleDiagnosisInput = { + baseline: 0.96, + direction: "maximize", + epsilon: 0.01, + lambda: 0.1, + experiments: ["d1", "d2", "d3"].map((experiment) => ({ + experiment, + outcome: experiment === chosen[0] ? { kind: "valid", score: 1.1 } : experiment === chosen[1] ? { kind: "insufficient" } : { kind: "untested" }, + })), + explanations: ruleInput(9, false).explanations, + }; + const settlement: DiagnosisSettlement = { + run: "run-1", + judged: { + experiments: [ + { + id: chosen[0]!, ending: "finished", wallSeconds: { status: "measured", value: 31.5 }, verdict: "valid", verdictReason: "校验器判合法", + score: { status: "measured", value: 1.1 }, metrics: { "sum-of-radii": { status: "measured", value: 1.1 }, [`${chosen[0]}.gap`]: { status: "measured", value: 0.02 } }, extra: null, + }, + ], + }, + dropped: [{ experiment: chosen[1]!, reason: "第二个实验两次核验都没落实" }], + input: settleInput, + output: settleDiagnosis(settleInput), + }; + l.append("diagnosisSettlement", { round: 1, result: settlement as never }); + return { l, w, chosen }; +} + +const mat = (l: Ledger, kind: SubmissionKind, actor: string, stageId: string, object: string | null, extra: { changes: CodeChanges | null; artifacts: Record> | null } = NONE, facts: StudyFacts = study) => + buildMaterial({ kind, actor, stageId, object, ledger: l, study: facts, prompts, codeRoot: "/code", ...extra }); + +test("诊 1：基线代码、基线那次运行的结果和答案、没有别人这一轮的诊断；上限取自配置（A、M）", (t) => { + const { l } = diagnosisStudy(t, "diagProposalOpen"); + const m = mat(l, "diagnosisProposal", "a2", "s1-dp", null, { changes: null, artifacts: { "run-0": { "solution.json": "{\"centers\": []}" } } }); + assert.deepEqual(m.checklist, { + kind: "diagnosisProposal", + maxExplanations: 2, + maxExperiments: 2, + takenMetricIds: ["sum-of-radii", "wall-seconds", "answer-path", "safeguard-triggers"], + }); + assert.deepEqual(m.readable, [{ version: "v-base", label: "本批基线", dir: "/code/v-base" }]); + has(m, "运行 run-0：任务给的原始代码；算数（校验器判合法）；分数 0.96"); + has(m, "run-0/solution.json"); + assert.equal(m.records.find((x) => x.id === "run-0/solution.json")!.text, "{\"centers\": []}"); + assert.ok(headings(m).includes("上一轮的收尾")); + assert.ok(!headings(m).includes("本轮诊断结果"), "诊断这一段还没走完，不放本轮诊断结果"); + lacks(m, "UNIQUE-A1"); // a1 在同一阶段已经交了，阶段没关，别人读不到 + lacks(m, "强起点"); +}); + +test("诊 2：合并后的实验标明作者、全部诊断解释、自己的解释另列并给对号清单；看不到别人在这一步交的数", (t) => { + const { l } = diagnosisStudy(t, "diagPredictionOpen"); + const m = mat(l, "diagnosisPrediction", "a2", "s1-dpred", null); + assert.deepEqual(m.checklist, { kind: "diagnosisPrediction", explanations: ["r1-a2-dx1"], experiments: ["d1", "d2", "d3"] }); + has(m, "#### d1 强起点（作者 a1、a2）"); + has(m, "动哪一处：solve.py 起点\n"); // 合并后的正文取作者里编号最小的那一份（a1） + lacks(m, "solve.py 起点（乙的写法）"); + has(m, "#### d3 放宽间隙（作者 a2）"); + has(m, "UNIQUE-A1"); + has(m, "UNIQUE-A2"); + assert.ok(m.sections.some((s) => s.heading === "你的诊断解释" && s.text.includes("r1-a2-dx1"))); + for (const x of SEALED.a1) lacks(m, x); // a1 已经交了诊断预测，阶段还开着 +}); + +test("诊断预测的数：关闭之后，也不进任何别人的材料（诊 2 本人、实现、提案都没有）", (t) => { + const { l } = diagnosisStudy(t, "settled"); + l.append("stageOpen", { stageId: "s1-prop", stage: "proposal", round: 1, roster: ["a1", "a2"] }); + const proposals = [mat(l, "proposal", "a1", "s1-prop", null), mat(l, "proposal", "a2", "s1-prop", null)]; + for (const m of proposals) { + for (const x of [...SEALED.a1, ...SEALED.a2]) lacks(m, x); + } + const own = mat(l, "diagnosisPrediction", "a1", "s1-dpred", null); + for (const x of SEALED.a2) lacks(own, x); +}); + +test("诊断实现：选中的实验原文、基线代码、原始产物的位置；不给信念本和成绩", (t) => { + const { l, chosen } = diagnosisStudy(t, "selected"); + const m = mat(l, "diagnosisImplementation", "x1", "s1-dimpl", D); + assert.deepEqual(m.checklist, { kind: "diagnosisImplementation", experiments: chosen }); + assert.deepEqual(m.readable, [{ version: "v-base", label: "起点代码（当前基线）", dir: "/code/v-base" }]); + has(m, "产出当前基线的那次正式运行（run-0）的原始产物已经只读地放进你的工作目录"); + for (const id of chosen) has(m, `### ${id} `); + assert.ok(!headings(m).includes("信念本") && !headings(m).includes("本批基线那次运行的结果")); + lacks(m, "UNIQUE-A2"); // 诊断解释不给实验员：只给选中的实验 +}); + +test("诊断修复：实现的材料加退回意见（没通过的实验、依据、上一次位置表和改动）、能读上一次的诊断程序；通过的实验不要动", (t) => { + const { l, chosen } = diagnosisStudy(t, "repairOpen"); + const m = mat(l, "diagnosisRepair", "x1", "s1-dimpl2", D, { changes: CHANGES_C1, artifacts: null }); + assert.deepEqual(m.checklist, { kind: "diagnosisRepair", experiments: chosen }); + assert.deepEqual(m.readable.map((x) => x.version), ["v-base", "v-diag"]); + const opinion = m.sections.find((s) => s.heading === "退回意见")!.text; + assert.ok(opinion.includes(`实验 ${chosen[1]}：没照写法做；solve.py:20`)); + assert.ok(!opinion.includes(`实验 ${chosen[0]}：`), "通过的实验不列"); + assert.ok(opinion.includes("核验员：第二个实验没照写法做")); + assert.ok(opinion.includes(`"experiment": "${chosen[0]}"`) && opinion.includes("多起点循环")); +}); + +test("诊断核验：选中的实验原文、基线到诊断程序的改动、位置表；不给信念本和实验员的说明", (t) => { + const { l, chosen } = diagnosisStudy(t, "implemented"); + const m = mat(l, "diagnosisVerification", "v1", "s1-dver", D, { changes: CHANGES_C1, artifacts: null }); + assert.deepEqual(m.checklist, { kind: "diagnosisVerification", experiments: chosen }); + assert.deepEqual(m.readable.map((x) => x.version), ["v-base", "v-diag"]); + has(m, "多起点循环"); + has(m, "solve.py:10-19"); + for (const id of chosen) has(m, `### ${id} `); + assert.ok(!headings(m).includes("信念本")); + lacks(m, "UNIQUE-X1"); +}); + +test("本轮诊断结果：提案材料里每个实验的做法、实测、用时、结局，每条解释确认或否定，推出的方向；排在基线之后、研究记录之前", (t) => { + const { l, chosen } = diagnosisStudy(t, "settled"); + l.append("stageOpen", { stageId: "s1-prop", stage: "proposal", round: 1, roster: ["a1", "a2"] }); + const m = mat(l, "proposal", "a1", "s1-prop", null); + const h = headings(m); + assert.ok(h.indexOf("基线") < h.indexOf("本轮诊断结果") && h.indexOf("本轮诊断结果") < h.indexOf("研究记录")); + const section = m.sections.find((s) => s.heading === "本轮诊断结果")!.text; + const [first, second] = chosen as [string, string]; + assert.ok(section.includes(`诊断运行 run-1：在基线 base（最终指标 0.96）上，一次运行做了 1 个实验，只扣 1 次`)); + const line = (id: string) => section.split("\n").find((x) => x.startsWith(`- ${id} `))!; + for (const part of ["实测 1.1，改进量 +0.14", "用时 31.5 秒", "其余指标：d1.gap 0.02", "测前合并范围"]) assert.ok(line(first!).includes(part), part); + assert.ok(line(second!).includes("证据不足：核验不通过，第二个实验两次核验都没落实")); + assert.match(section, /没做：一次诊断最多做 D 个实验，放不下/); + assert.match(section, /- r1-a1-dx1（作者 a1）：确认 .*；否定 /); + assert.ok(section.includes("甲的诊断解释：卡在起点 UNIQUE-A1") && section.includes("乙的诊断解释：卡在收尾 UNIQUE-A2")); + assert.match(section, /程序按诊断推出的方向：d\d（实测 \+0\.14）。这只是按公式推出来的参考，你可以不跟。/); + assert.ok(m.records.some((x) => x.id === "diagnosis-r1" && x.text.includes("run-1") && !x.text.includes("7.771"))); + assert.ok(m.readable.some((x) => x.version === "v-diag" && x.label === "第 1 轮诊断程序的代码")); +}); + +/** 只有基线、第 1 轮提案阶段开着的账本：本轮没有诊断记录。 */ +function bare(t: TestContext): Ledger { + const { ledger: l } = freshLedger(t); + const w = new Writer(l); + l.append("code", { object: "base", version: "v-base", locations: null }); + w.run("run-0", "baseline", 0, null, judged(0.96, "main")); + w.open("s1-prop", "proposal", 1, ["a1", "a2"]); + return l; +} +const diagnosisText = (m: TaskMaterial) => m.sections.find((s) => s.heading === "本轮诊断结果")!.text; + +test("本轮没有诊断：写原因。余额不到 2 次、配置关掉（余额够却没有决定）", (t) => { + const l = bare(t); + // 基线占 1 次：运行总数 2 时余额 1（不到 2），总数 10 时余额 9。 + assert.equal(diagnosisText(mat(l, "proposal", "a1", "s1-prop", null, NONE, { ...study, totalRuns: 2 })), "本轮没有诊断：余额只剩 1 次，不到 2 次，要留 1 次给本轮正向"); + assert.equal(diagnosisText(mat(l, "proposal", "a1", "s1-prop", null)), "本轮没有诊断：配置关掉了诊断"); +}); + +test("本轮没有诊断：信息价值不抵成本时，原因带最好的组合净值；选中的实验一个都没剩下时，原因取事件", (t) => { + const l = bare(t); + const w = new Writer(l); + const input = ruleInput(9, true); // 两条解释对每个实验预测一样：信息价值只剩开放预留那一点，抵不过成本 + const decision = selectDiagnosis(input); + assert.equal(decision.diagnose, false); + w.decision("selectDiagnosis", 1, input, decision); + const best = Math.max(...decision.combos.map((c) => c.net)); + const text = diagnosisText(mat(l, "proposal", "a1", "s1-prop", null)); + assert.equal(text, `本轮没有诊断：信息价值减成本都不大于 0，最好的组合净值 ${Number(best.toPrecision(4))}`); + + // 同一轮另一份账本：决定是做，但核验都没通过、修复用完。 + const l2 = bare(t); + const w2 = new Writer(l2); + const done = selectDiagnosis(ruleInput(9, false)); + w2.decision("selectDiagnosis", 1, ruleInput(9, false), done); + l2.append("event", diagnosisNotRunEvent(1, "d1、d2 都没通过核验")); + assert.equal(diagnosisText(mat(l2, "proposal", "a1", "s1-prop", null)), "本轮没有诊断：选中的实验核验都没通过、修复次数用完，一个都没跑：d1、d2 都没通过核验"); +}); + +test("诊断运行在诊断结算写入之前不公开，写入之后公开（和正向的采纳、消融的结算同一类规则）", (t) => { + const run = (l: Ledger) => new Board(l.records()).runs().find((x) => x.plan.run === "run-1")!; + const running = diagnosisStudy(t, "run").l; + assert.equal(new Board(running.records()).runPublic(run(running)), false); + const settled = diagnosisStudy(t, "settled").l; + assert.equal(new Board(settled.records()).runPublic(run(settled)), true); +}); + +test("信念本的诊断历史：各轮的实验、确认与否定、方向；战绩按人加一行诊断；诊断实验的实测不算手里最好", (t) => { + const { l, chosen } = diagnosisStudy(t, "settled"); + l.append("stageOpen", { stageId: "s1-prop", stage: "proposal", round: 1, roster: ["a1", "a2"] }); + const book = mat(l, "proposal", "a1", "s1-prop", null).sections.find((s) => s.heading === "信念本")!.text; + assert.ok(book.includes("**诊断历史**")); + assert.ok(book.includes(`- 第 1 轮：\n 诊断运行 run-1：在基线 base`)); + assert.ok(book.includes(` - ${chosen[0]} `) && book.includes("确认 d1；否定 无")); + assert.ok(book.includes("手里最好 0.96"), "诊断实验实测 1.1，但诊断的答案不采纳，手里最好仍是基线"); + // 战绩：a1、a2 各一条诊断：只有 d1 一个有效实验，样本各 1 个。a1 的 q5 = 1.25 > 实测 1.1 偏高；a2 的 q5 = 0.95 < 1.1 偏低。 + assert.ok(book.includes("- a1，诊断 1 次：中位数偏高 1、偏低 0")); + assert.ok(book.includes("- a2，诊断 1 次：中位数偏高 0、偏低 1")); +}); + +test("信念本的诊断历史：第 2 轮诊断关着时，第 1 轮的历史还在，第 2 轮没走完不列；第 2 轮诊断任务读不到第 2 轮的诊断结果", (t) => { + const { l } = diagnosisStudy(t, "settled"); + l.append("stageOpen", { stageId: "s1-prop", stage: "proposal", round: 1, roster: ["a1", "a2"] }); + l.append("stageClose", { stageId: "s1-prop" }); + // 第 1 轮采纳 r1-c1 作基线要写一大串记录，这里只看第 2 轮的诊断任务：基线仍是 base，所以直接开第 2 轮的诊断阶段。 + l.append("stageOpen", { stageId: "s2-dp", stage: "diagnosisProposal", round: 2, roster: ["a1", "a2"] }); + const m = mat(l, "diagnosisProposal", "a1", "s2-dp", null, { changes: null, artifacts: {} }); + assert.ok(!headings(m).includes("本轮诊断结果")); + const book = m.sections.find((s) => s.heading === "信念本")!.text; + assert.ok(book.includes("- 第 1 轮：") && !book.includes("- 第 2 轮：")); +}); + +test("实现正向候选的实验员：本轮有诊断时能读诊断程序（只读，注明冻结协议引用诊断实验时作参考）；没有诊断时没有", (t) => { + const { l } = diagnosisStudy(t, "settled"); + const w = new Writer(l); + l.append("stageOpen", { stageId: "s1-prop", stage: "proposal", round: 1, roster: ["a1", "a2"] }); + w.protocol("r1-c1", protocolOf("r1-c1", P_A1, 1, "base", ["r1-a1-1"], ["加多起点"])); + w.open("s1-impl-c1", "implementation", 1, ["x1"]); + const m = mat(l, "implementation", "x1", "s1-impl-c1", "r1-c1"); + assert.deepEqual(m.readable, [ + { version: "v-base", label: "起点代码（本批基线）", dir: "/code/v-base" }, + { version: "v-diag", label: "本轮诊断程序（只读，冻结协议引用诊断实验时作参考）", dir: "/code/v-diag" }, + ]); + has(m, "冻结协议引用诊断实验时作参考"); + lacks(m, "UNIQUE-A1"); // 只给代码，不给诊断解释和成绩 + lacks(m, "实测 1.1"); + + const l2 = bare(t); + const w2 = new Writer(l2); + w2.protocol("r1-c1", protocolOf("r1-c1", P_A1, 1, "base", ["r1-a1-1"], ["加多起点"])); + w2.open("s1-impl-c1", "implementation", 1, ["x1"]); + assert.deepEqual(mat(l2, "implementation", "x1", "s1-impl-c1", "r1-c1").readable.map((x) => x.version), ["v-base"]); +}); diff --git a/tests/materials-helpers.ts b/tests/materials-helpers.ts index 0f6305c..9619ac7 100644 --- a/tests/materials-helpers.ts +++ b/tests/materials-helpers.ts @@ -36,6 +36,7 @@ export const study: StudyFacts = { lambda: 0.1, totalRuns: 10, proposalLimit: 2, + diagnosisExperimentLimit: 2, factorLimit: 3, metrics: [ { id: "sum-of-radii", layer: "final", meaning: "半径和", unit: "无", direction: "maximize", howMeasured: "校验器", missing: "invalidateRun" }, @@ -114,7 +115,7 @@ export class Writer { decision(decision: string, round: number, input: unknown, output: unknown): void { this.l.append("decision", { decision, round, rule: "test", ruleVersion: 1, params: null, seed: null, input: input as never, output: output as never }); } - run(run: string, purpose: "baseline" | "forward" | "ablation", round: number, object: string | null, j: JudgedResult): void { + run(run: string, purpose: "baseline" | "forward" | "ablation" | "diagnosis", round: number, object: string | null, j: JudgedResult | null): void { this.l.append("runPlanned", { run, purpose, round, object, definition }); this.l.append("runStarted", { run }); this.l.append("runResult", { run, process: exited, judged: j }); diff --git a/tests/rules-fixtures-diagnosis.test.ts b/tests/rules-fixtures-diagnosis.test.ts new file mode 100644 index 0000000..283ab8e --- /dev/null +++ b/tests/rules-fixtures-diagnosis.test.ts @@ -0,0 +1,294 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { + diagnosisModel, + runDiagnosisRule, + selectDiagnosis, + settleDiagnosis, + trackRecord, + type DiagnosisExplanation, + type DiagnosisRuleInput, + type ExperimentOutcome, +} from "../src/rules/index.ts"; +import { close, loadFixture, scenarioPicker, type Scenario } from "./rules-fixtures-helpers.ts"; + +// 预期值来自 tests/fixtures/rules/gen_diagnosis.py：只照方法.md 6.12、6.7、6.9 的文字、用有理数写成的独立参考， +// 没有读过 src/rules/diagnosis.ts 和 value.ts（重新生成：python3 tests/fixtures/rules/gen_diagnosis.py）。 +// 场景里有手算的几个（opposite_lambda0 的 VOI = 0.1，same_lambda0 的 VOI = 0，说明写在场景的 note 里）。 + +const fixture = loadFixture("diagnosis.json"); +const pick = scenarioPicker(fixture); + +const EXACT = 1e-12; +const BITS = 1e-9; +/** 只做一个实验时的 VOI：实现用网格近似，参考是逐段闭式积分。 */ +const closeK1 = (actual: number, expected: number, what: string): void => + close(actual, expected, 1e-9 + 2e-4 * Math.abs(expected), what); + +type Ranges = Record; + +function explanationsOf(s: Scenario): DiagnosisExplanation[] { + return s.input.explanations.map((e: { id: string; author: string; ranges: Ranges }) => ({ + id: e.id, + author: e.author, + experiments: Object.entries(e.ranges).map(([experiment, range]) => ({ experiment, range })), + })); +} + +function ruleInput(s: Scenario): DiagnosisRuleInput { + return { + baseline: s.input.b, + direction: s.input.direction, + lambda: s.input.lambda, + experiments: s.input.experiments, + explanations: explanationsOf(s), + balance: s.input.balance, + batchCap: s.input.batchCap, + maxExperiments: s.input.maxExperiments, + u: s.input.u, + decay: s.input.decay, + }; +} + +/** 选实验场景：事前数字、组合表、选中的实验都和独立参考一致。 */ +function checkSelection(name: string): ReturnType { + const s = pick.get(name); + const got = selectDiagnosis(ruleInput(s)); + const e = s.expected; + for (const [d, v] of Object.entries(e.priorGains as Record)) { + close(got.priorGains.find((x) => x.experiment === d)!.gain, v, EXACT, `${name} ${d} 事前期望收益`); + } + close(got.g0, e.g0, EXACT, `${name} ĝ₀`); + e.weights.forEach((w: number, j: number) => close(got.weights[j]!, w, EXACT, `${name} 起始权重 ${j}`)); + assert.deepEqual(got.combos.map((c) => c.experiments), e.combos.map((c: any) => c.experiments), `${name} 组合列表`); + e.combos.forEach((c: any, i: number) => { + const g = got.combos[i]!; + if (c.voi === null) return; + const what = `${name} {${c.experiments}}`; + if (c.experiments.length === s.input.experiments.length) close(g.voi, c.voi, EXACT, `${what} VOI`); + else closeK1(g.voi, c.voi, `${what} VOI`); + assert.equal(g.rHat, c.rHat, `${what} R̂`); + close(g.mPrime, c.mPrime, EXACT, `${what} m′`); + close(g.cost, c.cost, EXACT, `${what} 成本`); + close(g.value, c.value, 1e-9 + 2e-4 * Math.abs(c.value), `${what} 价值`); + close(g.net, c.net, 1e-9 + 2e-4 * Math.abs(c.value), `${what} 净值`); + }); + if ("diagnose" in e) { + assert.equal(got.diagnose, e.diagnose, `${name} 做不做`); + assert.deepEqual(got.chosen, e.chosen, `${name} 选中的实验`); + } + return got; +} + +test("6.12 信息价值：两条解释对实验预测相反时为正（手算 0.1），预测一样时为 0", () => { + const opposite = checkSelection("opposite_lambda0"); + close(opposite.combos[0]!.voi, 0.1, EXACT, "手算：0.5 × 0.2 − 0"); + assert.equal(opposite.diagnose, true); + + const same = checkSelection("same_lambda0"); + close(same.combos[0]!.voi, 0, EXACT, "预测一样、范围都在 0 以上：做不做都不改变看法"); + assert.equal(same.diagnose, false, "净值 = −成本 < 0"); + assert.equal(same.experiments[0]!.reason, "value"); + assert.equal(same.chosenNet, 0); + + // λ = 0.1：预测一样时只剩开放预留带来的一小部分，比预测相反小得多。 + const oppositeReserve = checkSelection("opposite_lambda01"); + const sameReserve = checkSelection("same_lambda01"); + close(sameReserve.combos[0]!.voi, 0.0025, EXACT, "手算：λ × (E[max(0, U)] − E U) = 0.1 × 0.025"); + assert.ok(sameReserve.combos[0]!.voi > 0); + assert.ok(oppositeReserve.combos[0]!.voi > 10 * sameReserve.combos[0]!.voi); +}); + +test("6.12 价值减成本：不管做几个实验成本相同；R̂ = ⌈(余额 − 1) ÷ 每批上限⌉", () => { + const got = checkSelection("multi_all"); + // 手算：余额 9、K = 2：R̂ = ⌈8 ÷ 2⌉ = 4，m′ = 1 + 0.5 + 0.25 + 0.125 = 1.875； + // 成本 = 2 × 0.0525 × (3/4 − 2/3) = 0.00875（ĝ₀ = 0.0525 来自参考脚本的事前期望）。 + assert.equal(got.combos.length, 1, "实验数 3 ≤ D = 4：只有全做一个组合"); + assert.equal(got.combos[0]!.rHat, 4); + close(got.combos[0]!.mPrime, 1.875, EXACT, "m′"); + close(got.combos[0]!.cost, 0.00875, EXACT, "成本"); + assert.deepEqual(got.chosen, ["d1", "d2", "d3"]); + assert.ok(got.experiments.every((x) => x.done && x.reason === null && x.bestCombo === null)); +}); + +test("6.12 选实验：余额 < 2 时不诊断，没有可选组合", () => { + const got = checkSelection("balance_one"); + assert.deepEqual(got.combos, []); + assert.equal(got.diagnose, false); + assert.deepEqual(got.chosen, []); + assert.ok(got.experiments.every((x) => !x.done && x.bestCombo === null)); +}); + +test("6.12 选实验：实验数超过 D 时只比较装满的组合", () => { + const one = checkSelection("over_cap_d1"); + assert.equal(one.combos.length, 3, "C(3,1) 个单实验组合"); + assert.ok(one.combos.every((c) => c.experiments.length === 1), "没有空组合，也没有更大的组合"); + assert.deepEqual(one.chosen, ["d2"]); + // 没选中的实验：放不下，并附上含它的组合里净值最大的那个（这里就是它自己的单实验组合）。 + const d3 = one.experiments.find((x) => x.experiment === "d3")!; + assert.equal(d3.done, false); + assert.equal(d3.reason, "budget"); + assert.deepEqual(d3.bestCombo!.experiments, ["d3"]); + close(d3.bestCombo!.net, pick.get("over_cap_d1").expected.combos[2].net, 1e-9 + 2e-4 * 0.1, "d3 的净值"); + + const two = checkSelection("over_cap_d2"); + assert.deepEqual(two.combos.map((c) => c.experiments), [["d1", "d2"], ["d1", "d3"], ["d2", "d3"]]); + assert.equal(two.chosen.length, 2); + // 多做一个实验信息价值只会不变或变大：每个两实验组合的 VOI 不小于它包含的任一单实验（参考脚本的精确值）。 + const singles = pick.get("over_cap_d1").expected.combos as Array<{ experiments: string[]; voi: number }>; + for (const pair of two.combos) { + for (const d of pair.experiments) { + const single = singles.find((c) => c.experiments[0] === d)!.voi; + assert.ok(pair.voi >= single - (1e-9 + 2e-4 * single), `{${pair.experiments}} 的 VOI ${pair.voi} 小于 {${d}} 的 ${single}`); + } + } +}); + +test("6.12 选实验：平票按实验编号（自然序，d2 排在 d10 前面）；越小越好按改进量算", () => { + const tie = checkSelection("tie_by_id"); + assert.deepEqual(tie.chosen, ["d2"]); + assert.deepEqual(tie.experiments.find((x) => x.experiment === "d10")!.bestCombo!.experiments, ["d10"]); + checkSelection("minimize_all"); +}); + +test("登记表：选诊断实验只有一条规则，结果带规则名和版本；同样的输入算两次逐位相同", () => { + const s = pick.get("multi_all"); + const run = runDiagnosisRule("valueMinusCost", ruleInput(s)); + assert.deepEqual(run.rule, { name: "valueMinusCost", version: 1 }); + assert.deepEqual(JSON.parse(JSON.stringify(run)), JSON.parse(JSON.stringify({ rule: run.rule, ...selectDiagnosis(ruleInput(s)) }))); +}); + +test("成分模型：每条解释一个成分加开放预留；解释合起来 1 − λ，同一人平分", () => { + const s = pick.get("settle_mixed"); + const m = diagnosisModel({ ...ruleInput({ ...s, input: { ...s.input, balance: 9, batchCap: 2, maxExperiments: 4, u: 0.5, decay: 0.5 } }), }); + // 手算：researcher-1 一条 0.9 × 1/2，researcher-2 两条各 0.9 × 1/4，预留 0.1。 + assert.deepEqual(m.components, ["e1", "e2", "e3", "openReserve"]); + [0.45, 0.225, 0.225, 0.1].forEach((w, j) => close(m.weights[j]!, w, EXACT, `权重 ${j}`)); +}); + +type Settled = ReturnType; + +function settle(name: string): { got: Settled; e: any } { + const s = pick.get(name); + const outcomes = s.input.outcomes as Record; + const got = settleDiagnosis({ + baseline: s.input.b, + direction: s.input.direction, + epsilon: s.input.epsilon, + lambda: s.input.lambda, + experiments: Object.entries(outcomes).map(([experiment, o]) => ({ + experiment, + outcome: o.kind === "valid" ? { kind: "valid", score: o.score! } : ({ kind: o.kind } as ExperimentOutcome), + })), + explanations: explanationsOf(s), + }); + const e = s.expected; + assert.deepEqual(got.experiments.map((x) => x.outcome), Object.keys(e.experiments).map((d) => e.experiments[d].outcome), `${name} 结局`); + for (const row of got.experiments) { + const x = e.experiments[row.experiment]; + const what = `${name} ${row.experiment}`; + close(row.merged.q1, x.merged.q1, BITS, `${what} 合并一成`); + close(row.merged.q5, x.merged.q5, BITS, `${what} 合并一半`); + close(row.merged.q9, x.merged.q9, BITS, `${what} 合并九成`); + if (x.gain === undefined) { + // 证据不足、没做：不判、不记账。 + assert.equal(row.gain, null, `${what} 收益`); + assert.equal(row.worldBits, null, `${what} 世界比特`); + assert.equal(row.surprise, null, `${what} 惊讶`); + assert.equal(row.disagreement, null, `${what} 分歧量`); + } else { + close(row.gain!, x.gain, EXACT, `${what} 收益`); + close(row.worldBits!, x.worldBits, BITS, `${what} 世界比特`); + assert.equal(row.surprise!.cell, x.surprise.bin, `${what} 惊讶所在格`); + close(row.surprise!.prior, x.surprise.p, BITS, `${what} 事前把握`); + assert.equal(row.surprise!.bits.status, x.surprise.status === "ok" ? "measured" : "unmeasured"); + if (row.surprise!.bits.status === "measured") close(row.surprise!.bits.value, x.surprise.surprise, BITS, `${what} 惊讶`); + close(row.disagreement!, x.disagreement, BITS, `${what} 分歧量`); + } + } + // 确认与否定：只列有效实验。 + assert.deepEqual( + Object.fromEntries(got.judgments.map((j) => [j.id, Object.fromEntries(j.verdicts.map((v) => [v.experiment, v.verdict]))])), + e.judgments, + `${name} 确认与否定`, + ); + const dir = got.direction; + assert.equal(dir.weightsKept, e.direction.weightsKept, `${name} 权重是否保持`); + e.direction.weights.forEach((w: number, j: number) => close(dir.weights[j]!, w, EXACT, `${name} 新权重 ${j}`)); + for (const g of dir.gains) { + const x = e.direction.gains[g.experiment]; + close(g.gain, x.gain, EXACT, `${name} ${g.experiment} 方向收益`); + assert.equal(g.measured, x.measured, `${name} ${g.experiment} 是实测还是估计`); + } + assert.equal(dir.experiment, e.direction.experiment, `${name} 推出的方向`); + close(dir.gain, e.direction.gain, EXACT, `${name} 方向收益`); + assert.equal(dir.found, e.direction.found, `${name} 有没有指出方向`); + return { got, e }; +} + +test("6.12 结算：证据不足的实验不判、不记账；没做的实验也不判", () => { + const { got } = settle("settle_mixed"); + assert.deepEqual(got.experiments.map((x) => x.outcome), ["valid", "insufficient", "untested"]); + // d2、d3 没有有效分数：每条解释只有 d1 一行判定，没有比特。 + assert.ok(got.judgments.every((j) => j.verdicts.length === 1 && j.verdicts[0]!.experiment === "d1")); + assert.equal(got.experiments.filter((x) => x.worldBits !== null).length, 1); + // 手算：d1 实测 1.045 = b + 0.045，收益 +0.045；落在 e1 的 [1.02, 1.08] 里确认，e2 的 [1.00, 1.06] 里确认，e3 的 [0.98, 1.10] 里确认。 + assert.deepEqual(got.judgments.map((j) => j.verdicts[0]!.verdict), ["confirmed", "confirmed", "confirmed"]); +}); + +test("6.12 结算：两个有效实验、确认与否定、权重更新", () => { + const { got } = settle("settle_two_valid"); + // d2 实测 1.25，e2 的 [1.1, 1.3] 确认；e1 的 [0.95, 1.05]、e3 的 [0.8, 1.0] 否定。 + assert.deepEqual(got.judgments.find((j) => j.id === "e2")!.verdicts.find((v) => v.experiment === "d2")!.verdict, "confirmed"); + assert.deepEqual(got.judgments.find((j) => j.id === "e1")!.verdicts.find((v) => v.experiment === "d2")!.verdict, "denied"); + assert.deepEqual(got.judgments.find((j) => j.id === "e3")!.verdicts.find((v) => v.experiment === "d2")!.verdict, "denied"); +}); + +test("6.12 结算：全部实验收益不为正时，记没有指出能涨分的方向", () => { + const { got } = settle("settle_no_direction"); + assert.equal(got.direction.found, false); + assert.ok(got.direction.gains.every((g) => g.gain <= 0)); + // 手算：收益分别是 0.97 − 1 = −0.03、0.8 − 1 = −0.2、0.9 − 1 = −0.1，最大的是 d1 的 −0.03。 + assert.equal(got.direction.experiment, "d1"); + close(got.direction.gain, -0.03, EXACT, "最大收益"); +}); + +test("6.12 结算：没有有效实验、实测落在所有成分之外时保持起始权重", () => { + const none = settle("settle_all_insufficient"); + assert.equal(none.got.direction.weightsKept, false, "没有有效实验时没有密度可乘，权重原样（不算'无从更新'）"); + assert.ok(none.got.judgments.every((j) => j.verdicts.length === 0)); + const outside = settle("settle_outside_all"); + assert.equal(outside.got.direction.weightsKept, true); + assert.deepEqual(outside.got.direction.weights, diagnosisModel({ + baseline: 1, direction: "maximize", lambda: 0.1, experiments: ["d1", "d2", "d3"], + explanations: explanationsOf(pick.get("settle_outside_all")), + }).weights); +}); + +test("6.12 结算：越小越好按改进量 b − x", () => { + const { got } = settle("settle_minimize"); + close(got.experiments[0]!.gain!, 0.12, EXACT, "手算：1.0 − 0.88"); + close(got.experiments[1]!.gain!, -0.05, EXACT, "手算：1.0 − 1.05"); + assert.equal(got.direction.experiment, "d1"); +}); + +test("战绩：按人、按正向 / 诊断 / 消融分开数，次序是编号、正向、诊断、消融", () => { + // 手算：researcher-2 的诊断样本 (q1, q5, q9) = (0.1, 0.2, 0.3)，实测 0.35 在 q9 之上：中位数偏低、被穿过上沿、没包住、落点 aboveQ9。 + const range = { q1: 0.1, q5: 0.2, q9: 0.3 }; + const rows = trackRecord([ + { researcher: "researcher-2", phase: "ablation", range, value: 0.2 }, + { researcher: "researcher-2", phase: "diagnosis", range, value: 0.35 }, + { researcher: "researcher-1", phase: "diagnosis", range, value: 0.05 }, + { researcher: "researcher-2", phase: "forward", range, value: 0.25 }, + ]); + assert.deepEqual(rows.map((r) => `${r.researcher}/${r.phase}`), [ + "researcher-1/diagnosis", "researcher-2/forward", "researcher-2/diagnosis", "researcher-2/ablation", + ]); + const t = rows[2]!.tally; + assert.deepEqual([t.total, t.medianLow, t.medianHigh, t.aboveTop, t.belowBottom, t.covered], [1, 1, 0, 1, 0, 0]); + assert.equal(t.placements.aboveQ9, 1); +}); + +test("diagnosis.json 里的场景都被测到", () => { + assert.deepEqual(pick.unused(), []); +}); diff --git a/tests/submissions.test.ts b/tests/submissions.test.ts index 733b021..037ef89 100644 --- a/tests/submissions.test.ts +++ b/tests/submissions.test.ts @@ -2,6 +2,7 @@ import assert from "node:assert/strict"; import { test, type TestContext } from "node:test"; import { ablationOutcome, + diagnosisOutcome, protocolBody, receive, verificationOutcome, @@ -442,3 +443,138 @@ test("消融实现的代码存不成版本：退回", async () => { }); assert.match(reason, /代码存不成版本：bin\/x（符号链接/); }); + +// --------------------------------------------------------------------------- +// 诊断（方法.md 4.3 诊 1–诊 4） +// --------------------------------------------------------------------------- + +const DX_EXPERIMENT = (name: string, missing: "recordMissing" | "invalidateRun" = "recordMissing", ids: string[] = []) => ({ + name, + spot: "solve.py 的起点生成", + body: "换成更强的起点，交出答案", + metrics: ids.map((id) => ({ ...metric(id), missing })), +}); +const DX_PROPOSAL = (over: Partial> = {}): Submission<"diagnosisProposal"> => ({ + explanations: [{ body: "卡在起点" }, { body: "卡在收尾" }], + experiments: [DX_EXPERIMENT("强起点"), DX_EXPERIMENT("拿掉收尾", "recordMissing", ["raw-score"])], + ...over, +}); +const DX_LIST: ChecklistOf<"diagnosisProposal"> = { kind: "diagnosisProposal", maxExplanations: 2, maxExperiments: 2, takenMetricIds: ["sum-of-radii", "answer-path"] }; +const dxProposal = (content: Submission<"diagnosisProposal">): Incoming => ({ kind: "diagnosisProposal", content, checklist: DX_LIST, known: null, jevModel: MODEL }); + +test("诊断解释与实验：解释 1 至 M 条、实验 1 至 A 个；超过就退回", async () => { + const many = DX_PROPOSAL({ explanations: [{ body: "a" }, { body: "b" }, { body: "c" }], experiments: [DX_EXPERIMENT("1"), DX_EXPERIMENT("2"), DX_EXPERIMENT("3")] }); + const reason = await rejected(dxProposal(many)); + assert.match(reason, /交了 3 条诊断解释，最多 2 条/); + assert.match(reason, /交了 3 个诊断实验，最多 2 个/); +}); + +test("诊断解释与实验：登记的指标只能是 recordMissing；同一个实验里不重名、不占用已有编号", async () => { + const reason = await rejected( + dxProposal( + DX_PROPOSAL({ + experiments: [ + DX_EXPERIMENT("甲", "invalidateRun", ["raw-score"]), + { ...DX_EXPERIMENT("乙"), metrics: [metric("a"), metric("a"), metric("answer-path")] }, + ], + }), + ), + ); + assert.match(reason, /第 1 个实验的指标 raw-score 选了 invalidateRun：诊断实验里的指标只能选 recordMissing/); + assert.match(reason, /第 2 个实验的 metrics 里编号 a 写了不止一次/); + assert.match(reason, /第 2 个实验的指标编号 answer-path 已被占用/); +}); + +test("诊断解释与实验：对每个实验问冻结三问；两个实验用同一个指标短名没问题（冻结时各带自己实验的前缀）", async (t) => { + fakeJev(t, () => false); + const v = await receive(dxProposal(DX_PROPOSAL({ experiments: [DX_EXPERIMENT("甲", "recordMissing", ["x"]), DX_EXPERIMENT("乙", "recordMissing", ["x"])] }))); + assert.equal(v.accepted, true); + assert.equal(v.jev!.length, 6, "两个实验各三问"); + assert.deepEqual(v.jev!.map((j) => j.question), ["self_contradictory", "changes_measurement", "too_vague", "self_contradictory", "changes_measurement", "too_vague"]); +}); + +test("诊断解释与实验：Jev 判是就退回，原因写明第几个实验和哪一问", async (t) => { + fakeJev(t, (q) => q === "too_vague"); + const v = await receive(dxProposal(DX_PROPOSAL())); + assert.equal(v.accepted, false); + const reason = (v as Extract).reason; + assert.match(reason, /Jev 判出下面几处有问题/); + assert.match(reason, /- 第 1 个诊断实验（强起点）：too_vague 概率 0\.99/); + assert.match(reason, /- 第 2 个诊断实验（拿掉收尾）：too_vague 概率 0\.99/); +}); + +const DX_PRED_LIST: ChecklistOf<"diagnosisPrediction"> = { kind: "diagnosisPrediction", explanations: ["r1-dx1", "r1-dx2"], experiments: ["d1", "d2"] }; +const dxPred = (predictions: Array<[string, string, ReturnType]>): Incoming => ({ + kind: "diagnosisPrediction", + content: { predictions: predictions.map(([explanation, experiment, range]) => ({ explanation, experiment, range })), body: "理由" }, + checklist: DX_PRED_LIST, + known: null, + jevModel: MODEL, +}); + +test("诊断预测：自己每条解释 × 每个合并后的实验，不多不少；收下时不问 Jev", async () => { + const all: Array<[string, string, ReturnType]> = [ + ["r1-dx1", "d1", r(1, 1.1, 1.2)], ["r1-dx1", "d2", r(1, 1.1, 1.2)], ["r1-dx2", "d1", r(1, 1.1, 1.2)], ["r1-dx2", "d2", r(1, 1.1, 1.2)], + ]; + assert.deepEqual(await receive(dxPred(all)), { accepted: true, jev: null, protocol: null }); + const reason = await rejected(dxPred([all[0]!, all[0]!, all[1]!, ["r1-dx3", "d1", r(1, 1.1, 1.2)]])); + assert.match(reason, /predictions 的（解释:实验）应当是 r1-dx1:d1、r1-dx1:d2、r1-dx2:d1、r1-dx2:d2/); + assert.match(reason, /你写的 r1-dx3:d1 不在里面/); + assert.match(reason, /还缺 r1-dx2:d1、r1-dx2:d2/); + assert.match(reason, /r1-dx1:d1 写了不止一次/); +}); + +test("诊断预测：三个数要严格递增", async () => { + const reason = await rejected(dxPred([["r1-dx1", "d1", r(1, 1, 1.2)], ["r1-dx1", "d2", r(1, 1.1, 1.2)], ["r1-dx2", "d1", r(1, 1.1, 1.2)], ["r1-dx2", "d2", r(1.3, 1.2, 1.1)]])); + assert.match(reason, /r1-dx1:d1 的 range 的三个数要严格递增 q1 < q5 < q9，你写的是 1 \/ 1 \/ 1\.2/); + assert.match(reason, /r1-dx2:d2 的 range 的三个数要严格递增/); +}); + +const DX_CODE_BODY = { locations: [{ experiment: "d1", locations: ["solve.py:3-9"] }], concerns: "无", body: "无" }; +type DxCode = Extract["known"]["code"]; +const dxCode = (kind: "diagnosisImplementation" | "diagnosisRepair", code: DxCode): Incoming => + kind === "diagnosisImplementation" + ? { kind, content: DX_CODE_BODY, checklist: { kind, experiments: ["d1"] }, known: { code }, jevModel: MODEL } + : { kind, content: DX_CODE_BODY, checklist: { kind, experiments: ["d1"] }, known: { code }, jevModel: MODEL }; + +test("诊断实现、修复：改了不许改的文件、代码存不了就退回；有差异块时每块问一次五问", async (t) => { + for (const kind of ["diagnosisImplementation", "diagnosisRepair"] as const) { + assert.match(await rejected(dxCode(kind, { ...SAVED, protectedFiles: ["validate.py"] })), /改了不许改的文件：validate\.py/); + assert.match(await rejected(dxCode(kind, { saved: false, path: "bin/x", reason: "符号链接" })), /代码存不成版本：bin\/x（符号链接）/); + } + fakeJev(t, () => false); + const hunks = [ + { path: "solve.py", change: "modified" as const, text: "@@ -1 +1 @@\n-a\n+b" }, + { path: "solve.py", change: "modified" as const, text: "@@ -9 +9 @@\n-c\n+d" }, + ]; + const v = await receive(dxCode("diagnosisRepair", { ...SAVED, hunks })); + assert.equal(v.accepted, true); + assert.equal(v.jev!.length, 10, "两块各五问"); +}); + +test("诊断核验：逐个实验的结果覆盖全部选中的实验", async () => { + const verification = (ids: string[]): Incoming => ({ + kind: "diagnosisVerification", + content: { experiments: ids.map((experiment) => ({ experiment, followed: true, nothingElse: true, where: "solve.py:3" })), body: "无" }, + checklist: { kind: "diagnosisVerification", experiments: ["d1", "d2"] }, + known: null, + jevModel: MODEL, + }); + assert.deepEqual(await receive(verification(["d2", "d1"])), { accepted: true, jev: null, protocol: null }); + const reason = await rejected(verification(["d1", "d3"])); + assert.match(reason, /experiments 的实验应当是 d1、d2；你写的 d3 不在里面；还缺 d2/); +}); + +test("诊断核验通过没有：逐个实验判，一个不通过不影响别的实验", () => { + const v: Submission<"diagnosisVerification"> = { + experiments: [ + { experiment: "d1", followed: true, nothingElse: true, where: "a" }, + { experiment: "d2", followed: false, nothingElse: true, where: "没找到" }, + { experiment: "d3", followed: true, nothingElse: false, where: "顺手改了别处" }, + { experiment: "d4", followed: false, nothingElse: false, where: "都不对" }, + ], + body: "无", + }; + assert.deepEqual(diagnosisOutcome(v), { passed: false, failing: ["d2", "d3", "d4"], notFollowed: ["d2", "d4"], touchedElse: ["d3", "d4"] }); + assert.deepEqual(diagnosisOutcome({ experiments: [v.experiments[0]!], body: "无" }), { passed: true, failing: [], notFollowed: [], touchedElse: [] }); +}); diff --git a/tests/tasks-diagnosis.test.ts b/tests/tasks-diagnosis.test.ts new file mode 100644 index 0000000..3cb49a4 --- /dev/null +++ b/tests/tasks-diagnosis.test.ts @@ -0,0 +1,254 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { after, test, type TestContext } from "node:test"; +import { Runner } from "../src/runner/index.ts"; +import { + diagnosisRun, + diagnosisUnversionedNames, + judgeDiagnosis, + loadTask, + placeBaselineArtifact, + prepareDiagnosisJudging, + prepareDiagnosisRun, + type DiagnosisExperiment, + type DiagnosisJudged, +} from "../src/tasks/index.ts"; +import type { HypothesisMetric, Measured, ProcessResult, RunDefinition } from "../src/vocab/index.ts"; + +// 诊断运行的测试（任务书 18 第 1.4 节）：一次运行跑几个实验，经真实 Docker 在任务镜像里跑；没有 Docker 时失败，不跳过。 +// 镜像和容器用 inforsi-test-tasks-diag 开头的固定名字，与别的测试文件错开。 +// 判分的预期值是手算的 2 的幂有理数布局（同 tests/tasks.test.ts）：宽松布局半径和 26/128 = 0.203125， +// 相切布局 26/64 = 0.40625。 + +const TASK_DIR = join(import.meta.dirname, "..", "tasks", "circle_packing"); +const IMAGE_TAG = "inforsi-test-tasks-diag"; +/** 测试用的总时限：6 个实验各分到 4 秒。 */ +const TOTAL_SECONDS = 24; + +/** 任务目录抄一份，只把总时限改小，其余（Dockerfile、校验器、harness）原样，镜像编号也就相同。 */ +function shortTask() { + const dir = mkdtempSync(join(tmpdir(), "inforsi-tasks-diag-task-")); + cpSync(TASK_DIR, dir, { recursive: true }); + const facts = JSON.parse(readFileSync(join(dir, "task.json"), "utf8")); + facts.run.wallSeconds = TOTAL_SECONDS; + writeFileSync(join(dir, "task.json"), JSON.stringify(facts)); + return loadTask(dir); +} + +const task = shortTask(); +const runner = new Runner(null); +const digest = await runner.buildImage(task.dir, IMAGE_TAG); +after(() => { + rmSync(task.dir, { recursive: true, force: true }); + docker(["rmi", IMAGE_TAG]); +}); + +function docker(args: string[]): { status: number | null; stderr: string } { + return spawnSync("docker", args, { encoding: "utf8" }); +} + +function forceRemove(name: string): void { + const removed = docker(["rm", "-f", name]); + if (removed.status !== 0 && !removed.stderr.includes("No such container")) throw new Error(removed.stderr); +} + +function tempDir(t: TestContext, label: string): string { + const dir = mkdtempSync(join(tmpdir(), `inforsi-tasks-diag-${label}-`)); + t.after(() => rmSync(dir, { recursive: true, force: true })); + return dir; +} + +async function run(t: TestContext, suffix: string, definition: RunDefinition, dir: string): Promise { + const name = `inforsi-test-tasks-diag-${suffix}`; + forceRemove(name); + t.after(() => forceRemove(name)); + const outcome = await runner.start(name, definition, dir); + assert.deepEqual(outcome, { started: true }); + return runner.wait(name, tempDir(t, "log")); +} + +const m = (value: number): Measured => ({ status: "measured", value }); + +function rowLayout(radius: number, spacing: number, firstX: number): object { + const n = 26; + return { + centers: Array.from({ length: n }, (_, i) => [firstX + i * spacing, 0.5]), + radii: Array(n).fill(radius), + reported_sum: 99, + }; +} + +const metric = (id: string): HypothesisMetric => ({ id, meaning: "测试用", unit: "无", direction: "maximize", howMeasured: "测试用", missing: "recordMissing" }); + +/** 六个实验：分别测"拿到了原始产物""指标不串""崩了不影响后面""超时""原始产物只读""答案不合法"。 */ +const EXPERIMENTS: DiagnosisExperiment[] = [ + { id: "d-loose", metrics: [metric("d-loose.note")] }, + { id: "d-same", metrics: [metric("d-same.note")] }, + { id: "d-crash", metrics: [] }, + { id: "d-slow", metrics: [] }, + { id: "d-tamper", metrics: [] }, + { id: "d-outside", metrics: [] }, +]; + +const PROGRAM = `import json +import time + + +def layout(radius, spacing, first_x): + return ([[first_x + i * spacing, 0.5] for i in range(26)], [radius] * 26, 99.0) + + +def run_experiment(experiment_id, baseline_dir): + if experiment_id == "d-loose": + with open("metrics.json", "w") as handle: + json.dump({"d-loose.note": 0.5}, handle) + return layout(1 / 128, 1 / 32, 7 / 64) + if experiment_id == "d-same": + with open(baseline_dir + "/solution.json") as handle: + solution = json.load(handle) + return solution["centers"], solution["radii"], solution["reported_sum"] + if experiment_id == "d-crash": + raise RuntimeError("故意崩") + if experiment_id == "d-slow": + time.sleep(60) + if experiment_id == "d-tamper": + with open(baseline_dir + "/solution.json", "w") as handle: + handle.write("{}") + if experiment_id == "d-outside": + return layout(1 / 128, 1 / 32, 2.0) + raise RuntimeError("不认识的实验 " + experiment_id) +`; + +/** 基线那次正式运行的运行目录：只要有 solution.json。放的是相切布局，半径和 0.40625。 */ +function baselineRunDir(t: TestContext): string { + const dir = tempDir(t, "baseline-run"); + writeFileSync(join(dir, "solution.json"), JSON.stringify(rowLayout(1 / 64, 1 / 32, 7 / 64))); + return dir; +} + +function programDir(t: TestContext): string { + const dir = tempDir(t, "program"); + writeFileSync(join(dir, "initial.py"), PROGRAM); + return dir; +} + +function value(item: Measured | undefined): number { + assert.equal(item?.status, "measured", JSON.stringify(item)); + return (item as { value: number }).value; +} + +function reason(item: Measured | undefined): string { + assert.equal(item?.status, "unmeasured", JSON.stringify(item)); + return (item as { reason: string }).reason; +} + +test("运行定义：一个容器，总时限和核数照任务，实验按顺序、每个分到总时限 ÷ 实验数，种子经环境变量交给 harness", () => { + const definition = diagnosisRun(task, digest, "seed-x", ["a", "b", "c"]); + assert.equal(definition.wallSeconds, TOTAL_SECONDS); + assert.equal(definition.cores, 1); + assert.equal(definition.env.INFORSI_SEED, "seed-x"); + assert.deepEqual(definition.command, [ + "python", "-B", "/work/harness-diagnosis.py", "--program", "initial.py", "--baseline", "baseline-artifact", + "--answers", "answers", "--seconds-each", "8", "--report", "diagnosis.json", "--experiments", "a", "b", "c", + ]); +}); + +test("原始产物：只读地放进实验员工作目录，存版本时被跳过；运行目录里同样有，并带全部实验登记的指标说明", (t) => { + assert.deepEqual(diagnosisUnversionedNames(task), [...task.facts.unversionedNames, "baseline-artifact"]); + const workDir = tempDir(t, "work"); + placeBaselineArtifact(task, baselineRunDir(t), workDir); + const placed = join(workDir, "baseline-artifact", "solution.json"); + assert.equal(statSync(placed).mode & 0o777, 0o444); + assert.deepEqual(JSON.parse(readFileSync(placed, "utf8")), rowLayout(1 / 64, 1 / 32, 7 / 64)); + + const runDir = tempDir(t, "run"); + prepareDiagnosisRun(task, programDir(t), baselineRunDir(t), runDir, EXPERIMENTS); + for (const name of ["initial.py", "harness.py", "harness-diagnosis.py", "baseline-artifact/solution.json", "metrics-spec.json"]) { + assert.ok(existsSync(join(runDir, name)), name); + } + assert.ok(statSync(join(runDir, "answers")).isDirectory()); + const spec = JSON.parse(readFileSync(join(runDir, "metrics-spec.json"), "utf8")) as HypothesisMetric[]; + assert.deepEqual(spec.map((s) => s.id), ["d-loose.note", "d-same.note"]); +}); + +test("从头到尾：六个实验一个容器按顺序跑，崩了、超时、改只读文件的各记各的，不影响别人；逐个单独判分", async (t) => { + const baseline = baselineRunDir(t); + const runDir = tempDir(t, "e2e-run"); + prepareDiagnosisRun(task, programDir(t), baseline, runDir, EXPERIMENTS); + const ids = EXPERIMENTS.map((e) => e.id); + const process = await run(t, "e2e", diagnosisRun(task, digest, "inforsi-test-seed", ids), runDir); + assert.equal(process.ending, "exited", JSON.stringify(process)); + assert.equal((process as { exitCode: number }).exitCode, 0); + + const judgeDir = tempDir(t, "e2e-judge"); + const step = prepareDiagnosisJudging(task, runDir, judgeDir, digest, process, EXPERIMENTS); + assert.equal(step.status, "validate"); + if (step.status !== "validate") return; + const validator = await run(t, "e2e-validate", step.definition, judgeDir); + const judged = judgeDiagnosis(task, runDir, judgeDir, process, validator, EXPERIMENTS); + const by = Object.fromEntries(judged.experiments.map((e) => [e.id, e])); + assert.deepEqual(judged.experiments.map((e) => e.id), ids); + + assert.equal(by["d-loose"]!.ending, "finished"); + assert.equal(by["d-loose"]!.verdict, "valid"); + assert.equal(value(by["d-loose"]!.score), 0.203125); + assert.equal(value(by["d-loose"]!.metrics["d-loose.note"]), 0.5); + assert.equal(value(by["d-loose"]!.metrics["legalization-deduction"]), 0); + assert.ok(value(by["d-loose"]!.wallSeconds) > 0 && value(by["d-loose"]!.wallSeconds) < 4); + + // 读到了原始产物：原样返回，成绩就是那份产物的半径和。指标没写，记测不到，也没有串进上一个实验写的值。 + assert.equal(by["d-same"]!.verdict, "valid"); + assert.equal(value(by["d-same"]!.score), 0.40625); + assert.match(reason(by["d-same"]!.metrics["d-same.note"]), /metrics\.json 不存在/); + + assert.equal(by["d-crash"]!.ending, "crashed"); + assert.equal(by["d-crash"]!.verdict, "invalid"); + assert.match(reason(by["d-crash"]!.score), /非零退出码/); + assert.match(reason(by["d-crash"]!.metrics["raw-score"]), /没有跑校验器/); + + // 超时：被杀在分到的 4 秒上，用时不短于 4 秒；后面的实验照常跑完。 + assert.equal(by["d-slow"]!.ending, "timeout"); + assert.match(by["d-slow"]!.verdictReason, /超过分到的时间 4 秒/); + assert.ok(value(by["d-slow"]!.wallSeconds) >= 4 && value(by["d-slow"]!.wallSeconds) < 8); + + assert.equal(by["d-tamper"]!.ending, "crashed"); + assert.deepEqual(JSON.parse(readFileSync(join(runDir, "baseline-artifact", "solution.json"), "utf8")), rowLayout(1 / 64, 1 / 32, 7 / 64)); + + assert.equal(by["d-outside"]!.ending, "finished"); + assert.equal(by["d-outside"]!.verdict, "invalid"); + assert.match(by["d-outside"]!.verdictReason, /^校验器判不合法/); + assert.match(reason(by["d-outside"]!.score), /^不算数/); +}); + +/** 手写一次"只跑了一部分"的运行目录：report 里只有头几项。 */ +function partialRun(t: TestContext, report: object[]): string { + const dir = tempDir(t, "partial"); + writeFileSync(join(dir, "diagnosis.json"), JSON.stringify(report)); + return dir; +} + +test("整次运行被总时限杀掉：报告里没有的实验记没轮到，写明整次运行的结局；已有记录的照常", (t) => { + const timeout: ProcessResult = { ending: "timeout", cpuSeconds: m(23), wallSeconds: m(24), peakMemoryBytes: m(5) }; + const two = EXPERIMENTS.slice(0, 2); + const runDir = partialRun(t, [{ id: "d-loose", status: "timeout", wall_seconds: 4.01 }]); + const step = prepareDiagnosisJudging(task, runDir, tempDir(t, "judge"), digest, timeout, two); + assert.equal(step.status, "judged"); + const judged = (step as { result: DiagnosisJudged }).result; + assert.equal(judged.experiments[0]!.ending, "timeout"); + assert.equal(value(judged.experiments[0]!.wallSeconds), 4.01); + assert.equal(judged.experiments[1]!.ending, "notRun"); + assert.match(judged.experiments[1]!.verdictReason, /没轮到：整次诊断运行超过单次时限 24 秒/); + assert.match(reason(judged.experiments[1]!.wallSeconds), /没轮到/); + // 没有交出答案的实验，judgeDiagnosis 不要校验器的结果。 + assert.deepEqual(judgeDiagnosis(task, runDir, tempDir(t, "judge2"), timeout, null, two), judged); +}); + +test("运行正常退出、报告里却缺实验：报错停下，不当成候选的失败", (t) => { + const exited: ProcessResult = { ending: "exited", exitCode: 0, cpuSeconds: m(1), wallSeconds: m(2), peakMemoryBytes: m(5) }; + const runDir = partialRun(t, []); + mkdirSync(join(runDir, "answers")); + assert.throws(() => prepareDiagnosisJudging(task, runDir, tempDir(t, "judge"), digest, exited, EXPERIMENTS), /没有实验 d-loose/); +}); diff --git a/tests/vocab.test.ts b/tests/vocab.test.ts index 54d2a1a..0db5c7f 100644 --- a/tests/vocab.test.ts +++ b/tests/vocab.test.ts @@ -102,6 +102,14 @@ const samples: { [K in SubmissionKind]: Submission } = { ], }, claim: { choices: ["researcher-2-e1", "researcher-1-e2"] }, + diagnosisProposal: { + explanations: [{ body: "卡在起点：……；若换成明显更强的起点分数不涨，这条解释就不成立" }], + experiments: [{ name: "更强的起点", spot: "solve.py 的起点生成", body: "换成……，交出答案", metrics: metrics.defined }], + }, + diagnosisPrediction: { predictions: [{ explanation: "r2-dx1", experiment: "d1", range }], body: "理由" }, + diagnosisImplementation: { locations: [{ experiment: "d1", locations: ["solve.py:10-40"] }], concerns: "无", body: "说明" }, + diagnosisRepair: { locations: [{ experiment: "d1", locations: ["solve.py:10-40"] }], concerns: "无", body: "按核验意见补上" }, + diagnosisVerification: { experiments: [{ experiment: "d1", followed: true, nothingElse: true, where: "solve.py:10-40" }], body: "无" }, proposal: { factors, safeguards: ["final-shrink"], metrics, authorEstimate: range, body }, firstPrediction: { range, body: "理由" }, critique: { target: "p2", body: "（推测）扰动可能和起点数混杂" }, diff --git a/tests/workspace.test.ts b/tests/workspace.test.ts index d8aa720..340823c 100644 --- a/tests/workspace.test.ts +++ b/tests/workspace.test.ts @@ -260,3 +260,23 @@ test("比较的行号：改一行、加几行、只删不加，新旧两边的 // 给模型看的差异原文带 3 行上下文：三处改动相隔不到 6 行，合成一块。 assert.match(result.diff, /^@@ -1,10 \+1,12 @@\n a\n-b\n\+B\n c\n d\n e\n\+X\n\+Y\n\+Z\n f\n g\n-h\n i\n j\n$/m); }); + +test("诊断程序（任务书 18）：基线代码取出后放进只读的原始产物、改成诊断程序再存，产物被跳过，比较只看程序的改动，取出再存号不变", (t) => { + const repo = newRepo(t); + const baseline = save(repo, makeDir(t, { "initial.py": "a\nb\n", "evaluate.py": "e\n" }), []); + const workDir = checkout(t, repo, baseline); + // 原始产物放进实验员工作目录，设成只读（0444），与 tasks 的 placeBaselineArtifact 一样。 + mkdirSync(join(workDir, "baseline-artifact")); + writeFileSync(join(workDir, "baseline-artifact", "solution.json"), "{}"); + chmodSync(join(workDir, "baseline-artifact", "solution.json"), 0o444); + writeFileSync(join(workDir, "initial.py"), "a\nb\nexperiment\n"); + const excluded = ["__pycache__", "solution.json", "baseline-artifact"]; + const program = save(repo, workDir, excluded); + assert.notEqual(program, baseline); + const comparison = compareVersions(repo, baseline, program); + assert.deepEqual(comparison.files.map((f) => f.path), ["initial.py"]); + assert.deepEqual((comparison.files[0] as Extract).hunks, [{ from: { after: 2 }, to: { start: 3, end: 3 } }]); + const again = checkout(t, repo, program); + assert.equal(existsSync(join(again, "baseline-artifact")), false); + assert.equal(save(repo, again, excluded), program); +});