commit 3058f4c7440e02e6cf9a0d218db667b6d3e631f2 Author: iomgaa Date: Mon Jul 20 00:49:10 2026 -0400 chore: bootstrap project scaffolding Add architecture doc (research-wiki/ARCHITECTURE.md), CLAUDE.md with tiered SOP for Fable 5, adapted .claude skills/hooks/settings, package skeleton (src/polygateway), pyproject with import-linter contracts, Makefile, .env.example and smoke test. diff --git a/.claude/scripts/hooks/post-edit-quality.sh b/.claude/scripts/hooks/post-edit-quality.sh new file mode 100755 index 0000000..768081a --- /dev/null +++ b/.claude/scripts/hooks/post-edit-quality.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# ============================================================ +# post-edit-quality.sh +# Claude Code PostToolUse hook — 每次编辑 .py 文件后自动检查 +# +# 触发时机: Claude 每次执行 Write/Edit 后 +# 作用: 对被修改的文件执行复杂度和风格检查(确定性质量守卫) +# 退出码: 0 = 通过, 非0 = 报告问题(Claude 会看到 stderr 反馈) +# +# 注册: 见 .claude/settings.json → hooks.PostToolUse +# ============================================================ +set -euo pipefail + +# PolyGateway conda 环境(存在则优先其工具链) +PROJ_ENV="$HOME/miniconda3/envs/PolyGateway/bin" +[[ -d "$PROJ_ENV" ]] && export PATH="$PROJ_ENV:$PATH" + +INPUT=$(cat) +FILE_PATH=$(echo "$INPUT" | jq -r '.tool_input.file_path // .tool_input.path // ""') + +# 只检查 .py 文件 +if [[ ! "$FILE_PATH" == *.py ]]; then + exit 0 +fi + +# reference/ 只读区、外部路径不检查(写入本身由 pre-tool-guard 拦截) +case "$FILE_PATH" in + */reference/*|reference/*) exit 0 ;; +esac + +# 只检查存在的文件 +if [[ ! -f "$FILE_PATH" ]]; then + exit 0 +fi + +ERRORS="" +WARNINGS="" + +# ── 1. Ruff 格式 + lint 检查 ── +if command -v ruff &> /dev/null; then + RUFF_OUTPUT=$(ruff check "$FILE_PATH" 2>&1 || true) + if [[ -n "$RUFF_OUTPUT" ]]; then + ERRORS+="[ruff] 风格/lint 问题：\n$RUFF_OUTPUT\n\n" + fi +fi + +# ── 2. Radon 圈复杂度(只报告 C 级及以下) ── +if command -v radon &> /dev/null; then + RADON_OUTPUT=$(radon cc "$FILE_PATH" -n C -s 2>&1 || true) + if echo "$RADON_OUTPUT" | grep -qE '^\s+[FMC]\s'; then + ERRORS+="[radon] 圈复杂度过高（≥C）：\n$RADON_OUTPUT\n\n" + fi +fi + +# ── 3. 文件行数检查(warning,不阻塞)── +LINE_COUNT=$(wc -l < "$FILE_PATH") +if [[ "$LINE_COUNT" -gt 200 ]]; then + WARNINGS+="[行数] $FILE_PATH 有 ${LINE_COUNT} 行，超过 200 行建议上限。\n\n" +fi + +# ── 4. 禁止裸 except ── +if grep -nE '^\s*except\s*:\s*$|^\s*except\s+Exception\s*:\s*pass' "$FILE_PATH" 2>/dev/null; then + ERRORS+="[安全] 检测到裸 except 或 except Exception: pass，请捕获具体异常类型（CLAUDE.md P5）。\n\n" +fi + +# ── 5. 禁止硬编码敏感信息 ── +if grep -nEi "(api_key|secret|password|token)\s*=\s*[\"'][^\"']+[\"']" "$FILE_PATH" 2>/dev/null; then + ERRORS+="[安全] 疑似硬编码敏感信息，请使用环境变量或 .env 文件。\n\n" +fi + +# ── 输出结果 ── +if [[ -n "$WARNINGS" ]]; then + echo -e "⚠️ Warnings（$FILE_PATH，不阻塞）：\n" >&2 + echo -e "$WARNINGS" >&2 +fi + +if [[ -n "$ERRORS" ]]; then + echo -e "❌ 代码质量检查发现问题（$FILE_PATH）：\n" >&2 + echo -e "$ERRORS" >&2 + exit 1 +fi + +exit 0 diff --git a/.claude/scripts/hooks/pre-commit-guard.sh b/.claude/scripts/hooks/pre-commit-guard.sh new file mode 100755 index 0000000..10ea7b0 --- /dev/null +++ b/.claude/scripts/hooks/pre-commit-guard.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# ============================================================ +# pre-commit-guard.sh +# Claude Code PreToolUse hook — 拦截 git commit,执行全量检查 +# +# 触发时机: Claude 尝试执行 git commit 之前 +# 作用: 阻塞提交直到质量门通过(确定性质量守卫) +# 退出码: 0 = 放行, 2 = 阻塞(Claude 必须先修复问题) +# +# 注册: 见 .claude/settings.json → hooks.PreToolUse (matcher: Bash) +# ============================================================ +set -euo pipefail + +# PolyGateway conda 环境(存在则优先其工具链) +PROJ_ENV="$HOME/miniconda3/envs/PolyGateway/bin" +[[ -d "$PROJ_ENV" ]] && export PATH="$PROJ_ENV:$PATH" + +INPUT=$(cat) +COMMAND=$(echo "$INPUT" | jq -r '.tool_input.command // ""') + +# 只拦截 git commit 命令 +if ! echo "$COMMAND" | grep -qE '(^|\s|&&|;)\s*git\s+commit'; then + exit 0 +fi + +ERRORS="" +WARNINGS="" + +CODE_DIRS=("src") + +for CODE_DIR in "${CODE_DIRS[@]}"; do + [[ -d "$CODE_DIR" ]] || continue + + # ── 1. 全量代码质量检查 ── + if command -v ruff &> /dev/null; then + RUFF_OUTPUT=$(ruff check "$CODE_DIR" 2>&1 || true) + if [[ -n "$RUFF_OUTPUT" ]]; then + ERROR_COUNT=$(echo "$RUFF_OUTPUT" | wc -l) + ERRORS+="[ruff] $CODE_DIR/ 中有 ${ERROR_COUNT} 个问题。运行 ruff check $CODE_DIR/ 查看详情。\n" + fi + fi + + if command -v radon &> /dev/null; then + RADON_OUTPUT=$(radon cc "$CODE_DIR" -n C -s 2>&1 || true) + if echo "$RADON_OUTPUT" | grep -qE '^\s+[FMC]\s'; then + ERRORS+="[radon] 存在圈复杂度 ≥ C 的函数：\n$RADON_OUTPUT\n" + fi + fi + + # ── 2. 文件行数检查 ── + while IFS= read -r pyfile; do + lines=$(wc -l < "$pyfile") + if [[ "$lines" -gt 200 ]]; then + WARNINGS+="[行数] $pyfile 有 ${lines} 行，超过 200 行建议上限。\n" + fi + done < <(find "$CODE_DIR" -name "*.py" 2>/dev/null || true) +done + +# ── 3. 测试检查 ── +if [[ -d tests ]] && command -v pytest &> /dev/null; then + if ! TEST_OUTPUT=$(pytest tests/ --tb=line -q 2>&1); then + FAILED=$(echo "$TEST_OUTPUT" | tail -3) + ERRORS+="[测试] 有测试未通过：\n$FAILED\n" + fi +fi + +# ── 判定结果 ── +if [[ -n "$WARNINGS" ]]; then + echo -e "⚠️ Warnings（不阻塞）：\n" >&2 + echo -e "$WARNINGS" >&2 +fi + +if [[ -n "$ERRORS" ]]; then + echo -e "❌ 提交被阻塞 — 请先修复以下问题：\n" >&2 + echo -e "$ERRORS" >&2 + echo -e "修复完成后重新执行 git commit。" >&2 + exit 2 +fi + +echo "✅ 所有检查通过，允许提交。" +exit 0 diff --git a/.claude/scripts/hooks/pre-tool-guard.sh b/.claude/scripts/hooks/pre-tool-guard.sh new file mode 100755 index 0000000..024f03d --- /dev/null +++ b/.claude/scripts/hooks/pre-tool-guard.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# ============================================================ +# pre-tool-guard.sh +# Claude Code PreToolUse hook — 硬边界安全守卫(确定性拦截) +# +# 拦截范围(exit 2 = 阻断,stderr 反馈给 Claude): +# 1. 对 reference/ 的任何写操作(只读铁律,文件工具 + Bash 双路拦截) +# 2. rm -rf 作用于项目根 / reference/ / 家目录 / 根目录 / 通配全删 +# 3. git push --force / -f 与 git push/commit --no-verify +# 4. 明显会泄露密钥的 .env 读取/外传命令(.env.example 除外) +# +# 定位: 本脚本只做确定性安全拦截,不做流程编排。 +# 注册: 见 .claude/settings.json → hooks.PreToolUse +# ============================================================ +set -euo pipefail + +INPUT=$(cat) +TOOL_NAME=$(echo "$INPUT" | jq -r '.tool_name // ""') + +block() { + echo -e "🚫 已被 pre-tool-guard 硬边界拦截：$1" >&2 + exit 2 +} + +# ── A. 文件写入类工具: reference/ 只读 ── +case "$TOOL_NAME" in + Write|Edit|MultiEdit|NotebookEdit) + FILE_PATH=$(echo "$INPUT" | jq -r '.tool_input.file_path // .tool_input.notebook_path // .tool_input.path // ""') + case "$FILE_PATH" in + */reference/*|reference/*) + block "reference/ 是只读参考区（CLAUDE.md 铁律），禁止写入: $FILE_PATH" + ;; + esac + exit 0 + ;; + Bash) : ;; # 继续走下方 Bash 命令检查 + *) exit 0 ;; +esac + +COMMAND=$(echo "$INPUT" | jq -r '.tool_input.command // ""') +[[ -z "$COMMAND" ]] && exit 0 + +# ── B. Bash: 对 reference/ 的写操作 ── +if echo "$COMMAND" | grep -qE '(^|/| )reference/'; then + # 写指示词: 删除/移动/覆盖/重定向/就地修改/权限变更 + if echo "$COMMAND" | grep -qE '(^|[;&|]\s*|\s)(rm|mv|chmod|chown|truncate|shred|unlink|rmdir)\s[^;&|]*reference/' \ + || echo "$COMMAND" | grep -qE '(cp|rsync|install|ln|tee|touch|mkdir)\s[^;&|]*\s[^;&|]*reference/' \ + || echo "$COMMAND" | grep -qE '>>?\s*[^ ;&|]*reference/' \ + || echo "$COMMAND" | grep -qE 'sed\s+(-[a-zA-Z]*\s+)*-i[^;&|]*reference/' \ + || echo "$COMMAND" | grep -qE 'git\s+[^;&|]*\b(checkout|restore|clean|reset|stash|apply|am)\b[^;&|]*reference/'; then + block "reference/ 是只读参考区（CLAUDE.md 铁律），禁止任何写/删/改操作。命令: $COMMAND" + fi +fi + +# ── C. 危险 rm -rf ── +if echo "$COMMAND" | grep -qE '(^|[;&|]\s*|\s)rm\s+(-[a-zA-Z]*[rR][a-zA-Z]*f[a-zA-Z]*|-[a-zA-Z]*f[a-zA-Z]*[rR][a-zA-Z]*|-[rR]\s+-f|-f\s+-[rR])\b'; then + # 提取 rm 之后的目标(粗粒度即可,宁可错杀) + if echo "$COMMAND" | grep -qE 'rm\s+(-[a-zA-Z]+\s+)*("?\$HOME"?|~|/|\.|\.\.|\*|"?\$\(pwd\)"?)(\s|$|/\*|"$)' \ + || echo "$COMMAND" | grep -qE 'rm\s+(-[a-zA-Z]+\s+)*[^ ]*reference/?(\s|$|\*)' \ + || echo "$COMMAND" | grep -qE 'rm\s+(-[a-zA-Z]+\s+)*[^ ]*PolyGateway/?(\s|$)'; then + block "危险的 rm -rf（项目根 / reference / 家目录 / 通配全删）。请精确指定要删除的具体文件。命令: $COMMAND" + fi +fi + +# ── D. git 危险操作 ── +if echo "$COMMAND" | grep -qE 'git\s+push[^;&|]*(--force\b|--force-with-lease\b|\s-f\b)'; then + block "禁止 git push --force（含 --force-with-lease）。如确需覆写远端历史，请人类手动执行。命令: $COMMAND" +fi +if echo "$COMMAND" | grep -qE 'git\s+(commit|push|merge)[^;&|]*--no-verify\b'; then + block "禁止 --no-verify 绕过提交检查（质量门是硬边界）。命令: $COMMAND" +fi + +# ── E. .env 密钥泄露 ── +# 允许 .env.example / .env.template;拦截对真实 .env 的直接读取与外传 +ENV_RE='(^|[ /"'"'"'=@,:])\.env(\.[a-z]+)?' +if echo "$COMMAND" | grep -qE "$ENV_RE" \ + && ! echo "$COMMAND" | grep -qE '\.env\.(example|template|sample)\b'; then + if echo "$COMMAND" | grep -qE '(^|[;&|]\s*|\s)(cat|less|more|head|tail|bat|strings|base64|xxd|od|nl)\s[^;&|]*\.env' \ + || echo "$COMMAND" | grep -qE '(^|[;&|]\s*|\s)(curl|wget|nc|scp|rsync|ftp)\s[^;&|]*\.env' \ + || echo "$COMMAND" | grep -qE '(^|[;&|]\s*|\s)(cp|mv)\s[^;&|]*\.env[^;&|]*\s[^;&|]*(reference|/tmp|/Users/Shared)' \ + || echo "$COMMAND" | grep -qE 'git\s+add\s[^;&|]*\.env'; then + block "疑似泄露/外传 .env 密钥的命令（读取密钥请通过 pydantic-settings 在代码内完成；确需查看请人类手动执行）。命令: $COMMAND" + fi +fi + +exit 0 diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 0000000..7c46fed --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,39 @@ +{ + "hooks": { + "PreToolUse": [ + { + "matcher": "Bash", + "hooks": [ + { + "type": "command", + "command": "bash \"$CLAUDE_PROJECT_DIR\"/.claude/scripts/hooks/pre-tool-guard.sh" + }, + { + "type": "command", + "command": "bash \"$CLAUDE_PROJECT_DIR\"/.claude/scripts/hooks/pre-commit-guard.sh" + } + ] + }, + { + "matcher": "Write|Edit|MultiEdit|NotebookEdit", + "hooks": [ + { + "type": "command", + "command": "bash \"$CLAUDE_PROJECT_DIR\"/.claude/scripts/hooks/pre-tool-guard.sh" + } + ] + } + ], + "PostToolUse": [ + { + "matcher": "Write|Edit|MultiEdit", + "hooks": [ + { + "type": "command", + "command": "bash \"$CLAUDE_PROJECT_DIR\"/.claude/scripts/hooks/post-edit-quality.sh" + } + ] + } + ] + } +} diff --git a/.claude/skills/brainstorming/SKILL.md b/.claude/skills/brainstorming/SKILL.md new file mode 100644 index 0000000..6f9562d --- /dev/null +++ b/.claude/skills/brainstorming/SKILL.md @@ -0,0 +1,51 @@ +--- +name: brainstorming +description: "Use before design-level creative work. MANDATORY when the change touches public API, port (Protocol) signatures, or architecture boundaries — these require an approved design and a human gate. For smaller changes (internal implementation, test fixes, tasks inside an approved plan), use your judgment: skip if requirements are already unambiguous." +--- + +# Brainstorming Ideas Into Designs + +把想法变成经过对比与确认的设计。产出是一份设计文档,不是代码。 + +## 触发边界(何时必须用) + +| 情形 | 是否强制 | +|---|---| +| 变更公共 API、端口(Protocol)签名、架构边界(`ports.py`/`types.py`/`errors.py`、middleware 洋葱层次、依赖铁律) | **强制**,且设计必须经人类确认后才能实施 | +| 新增子系统/新组件、里程碑级功能 | **强制** | +| 内部实现、测试修复、已批准 plan 内的任务、文档修订 | 自判;需求无歧义即可直接做 | + +不确定属于哪一类时,按"是否会改变库对下游三个项目的承诺"判断:会,就走设计。 + +## 核心要求(设计的验收标准) + +一份合格的设计必须包含,缺一即不完整: + +1. **备选方案对比**:至少 2-3 个可行方案、各自权衡、你的推荐及理由。只有一个方案 = 没有做设计。 +2. **旧版行为审计(仅重写/迁移类任务)**:本库大量工作是从 `reference/` 三项目迁移治理代码。凡替换/重写既有模块,必须列出旧版全部行为(含持久化、崩溃恢复、幂等、断点续跑),逐条标注"保留 / 替换 / 有意放弃"。未声明的隐式丢弃 = bug。 +3. **非功能维度(逐条回答,允许"不适用"但必须写明)**: + - 并发与取消:并发调用下的行为?`CancelledError` 穿透路径? + - 降级方向:依赖的后端不可用时,静默降级还是报错?(对照 CLAUDE.md 库铁律) + - 幂等与重复:同一操作重复执行是否安全? + - 持久化与原子性:什么时候落盘?部分写入会不会损坏数据? +4. **错误处理与测试策略**:失败落入哪个错误分类?怎么测? + +## 过程建议(非脚本) + +先查 `research-wiki/ARCHITECTURE.md` 与相关 `reference/` 代码,再提问;问题一次一个、能选择题就选择题;范围过大(多个独立子系统)先拆分再逐个设计。不做任务外的重构与抽象——设计只服务当前目标。 + +如需向人类展示视觉对比(布局/图示),可参考 `visual-companion.md`(可选工具,非流程)。 + +## 留痕与审批门(不可省略) + +1. **写设计文档**: `research-wiki/designs/YYYY-MM-DD--design.md`(≤400 行),并提交 git。 +2. **自审后送 Codex 独立审**: 用 `/codex:rescue --fresh --wait` 只读审查(需求覆盖、内部一致性、与 ARCHITECTURE.md/CLAUDE.md 的冲突)。逐条核验其意见后就地修订——不盲从。模板见 `spec-document-reviewer-prompt.md`。 +3. **人类审批门**: 凡触发"强制"档的设计,必须请人类审阅设计文档并明确同意后才进入 `writing-plans`。这是硬门,不因任何理由跳过。 +4. **Wiki 注册**(`research-wiki/` 存在时): + ```bash + .claude/tools/research_wiki.py add_entity research-wiki/ --type design --id --title "" + .claude/tools/research_wiki.py rebuild_index research-wiki/ + ``` + 在生成页中记录:选定方案、关键理由、**被否决的备选及否决原因**。 + +设计获批后的下一步是 `writing-plans`(若达到其触发规模),不要跳到实现类 skill。 diff --git a/.claude/skills/brainstorming/spec-document-reviewer-prompt.md b/.claude/skills/brainstorming/spec-document-reviewer-prompt.md new file mode 100644 index 0000000..f9b0e2e --- /dev/null +++ b/.claude/skills/brainstorming/spec-document-reviewer-prompt.md @@ -0,0 +1,49 @@ +# Spec Document Reviewer Prompt Template + +Use this template when dispatching a spec document reviewer subagent. + +**Purpose:** Verify the spec is complete, consistent, and ready for implementation planning. + +**Dispatch after:** Spec document is written to docs/superpowers/specs/ + +``` +Task tool (general-purpose): + description: "Review spec document" + prompt: | + You are a spec document reviewer. Verify this spec is complete and ready for planning. + + **Spec to review:** [SPEC_FILE_PATH] + + ## What to Check + + | Category | What to Look For | + |----------|------------------| + | Completeness | TODOs, placeholders, "TBD", incomplete sections | + | Consistency | Internal contradictions, conflicting requirements | + | Clarity | Requirements ambiguous enough to cause someone to build the wrong thing | + | Scope | Focused enough for a single plan — not covering multiple independent subsystems | + | YAGNI | Unrequested features, over-engineering | + + ## Calibration + + **Only flag issues that would cause real problems during implementation planning.** + A missing section, a contradiction, or a requirement so ambiguous it could be + interpreted two different ways — those are issues. Minor wording improvements, + stylistic preferences, and "sections less detailed than others" are not. + + Approve unless there are serious gaps that would lead to a flawed plan. + + ## Output Format + + ## Spec Review + + **Status:** Approved | Issues Found + + **Issues (if any):** + - [Section X]: [specific issue] - [why it matters for planning] + + **Recommendations (advisory, do not block approval):** + - [suggestions for improvement] +``` + +**Reviewer returns:** Status, Issues (if any), Recommendations \ No newline at end of file diff --git a/.claude/skills/brainstorming/visual-companion.md b/.claude/skills/brainstorming/visual-companion.md new file mode 100644 index 0000000..89ebc6a --- /dev/null +++ b/.claude/skills/brainstorming/visual-companion.md @@ -0,0 +1,287 @@ +# Visual Companion Guide + +Browser-based visual brainstorming companion for showing mockups, diagrams, and options. + +## When to Use + +Decide per-question, not per-session. The test: **would the user understand this better by seeing it than reading it?** + +**Use the browser** when the content itself is visual: + +- **UI mockups** — wireframes, layouts, navigation structures, component designs +- **Architecture diagrams** — system components, data flow, relationship maps +- **Side-by-side visual comparisons** — comparing two layouts, two color schemes, two design directions +- **Design polish** — when the question is about look and feel, spacing, visual hierarchy +- **Spatial relationships** — state machines, flowcharts, entity relationships rendered as diagrams + +**Use the terminal** when the content is text or tabular: + +- **Requirements and scope questions** — "what does X mean?", "which features are in scope?" +- **Conceptual A/B/C choices** — picking between approaches described in words +- **Tradeoff lists** — pros/cons, comparison tables +- **Technical decisions** — API design, data modeling, architectural approach selection +- **Clarifying questions** — anything where the answer is words, not a visual preference + +A question *about* a UI topic is not automatically a visual question. "What kind of wizard do you want?" is conceptual — use the terminal. "Which of these wizard layouts feels right?" is visual — use the browser. + +## How It Works + +The server watches a directory for HTML files and serves the newest one to the browser. You write HTML content to `screen_dir`, the user sees it in their browser and can click to select options. Selections are recorded to `state_dir/events` that you read on your next turn. + +**Content fragments vs full documents:** If your HTML file starts with `<!DOCTYPE` or `<html`, the server serves it as-is (just injects the helper script). Otherwise, the server automatically wraps your content in the frame template — adding the header, CSS theme, selection indicator, and all interactive infrastructure. **Write content fragments by default.** Only write full documents when you need complete control over the page. + +## Starting a Session + +```bash +# Start server with persistence (mockups saved to project) +scripts/start-server.sh --project-dir /path/to/project + +# Returns: {"type":"server-started","port":52341,"url":"http://localhost:52341", +# "screen_dir":"/path/to/project/.superpowers/brainstorm/12345-1706000000/content", +# "state_dir":"/path/to/project/.superpowers/brainstorm/12345-1706000000/state"} +``` + +Save `screen_dir` and `state_dir` from the response. Tell user to open the URL. + +**Finding connection info:** The server writes its startup JSON to `$STATE_DIR/server-info`. If you launched the server in the background and didn't capture stdout, read that file to get the URL and port. When using `--project-dir`, check `<project>/.superpowers/brainstorm/` for the session directory. + +**Note:** Pass the project root as `--project-dir` so mockups persist in `.superpowers/brainstorm/` and survive server restarts. Without it, files go to `/tmp` and get cleaned up. Remind the user to add `.superpowers/` to `.gitignore` if it's not already there. + +**Launching the server by platform:** + +**Claude Code (macOS / Linux):** +```bash +# Default mode works — the script backgrounds the server itself +scripts/start-server.sh --project-dir /path/to/project +``` + +**Claude Code (Windows):** +```bash +# Windows auto-detects and uses foreground mode, which blocks the tool call. +# Use run_in_background: true on the Bash tool call so the server survives +# across conversation turns. +scripts/start-server.sh --project-dir /path/to/project +``` +When calling this via the Bash tool, set `run_in_background: true`. Then read `$STATE_DIR/server-info` on the next turn to get the URL and port. + +**Codex:** +```bash +# Codex reaps background processes. The script auto-detects CODEX_CI and +# switches to foreground mode. Run it normally — no extra flags needed. +scripts/start-server.sh --project-dir /path/to/project +``` + +**Gemini CLI:** +```bash +# Use --foreground and set is_background: true on your shell tool call +# so the process survives across turns +scripts/start-server.sh --project-dir /path/to/project --foreground +``` + +**Other environments:** The server must keep running in the background across conversation turns. If your environment reaps detached processes, use `--foreground` and launch the command with your platform's background execution mechanism. + +If the URL is unreachable from your browser (common in remote/containerized setups), bind a non-loopback host: + +```bash +scripts/start-server.sh \ + --project-dir /path/to/project \ + --host 0.0.0.0 \ + --url-host localhost +``` + +Use `--url-host` to control what hostname is printed in the returned URL JSON. + +## The Loop + +1. **Check server is alive**, then **write HTML** to a new file in `screen_dir`: + - Before each write, check that `$STATE_DIR/server-info` exists. If it doesn't (or `$STATE_DIR/server-stopped` exists), the server has shut down — restart it with `start-server.sh` before continuing. The server auto-exits after 30 minutes of inactivity. + - Use semantic filenames: `platform.html`, `visual-style.html`, `layout.html` + - **Never reuse filenames** — each screen gets a fresh file + - Use Write tool — **never use cat/heredoc** (dumps noise into terminal) + - Server automatically serves the newest file + +2. **Tell user what to expect and end your turn:** + - Remind them of the URL (every step, not just first) + - Give a brief text summary of what's on screen (e.g., "Showing 3 layout options for the homepage") + - Ask them to respond in the terminal: "Take a look and let me know what you think. Click to select an option if you'd like." + +3. **On your next turn** — after the user responds in the terminal: + - Read `$STATE_DIR/events` if it exists — this contains the user's browser interactions (clicks, selections) as JSON lines + - Merge with the user's terminal text to get the full picture + - The terminal message is the primary feedback; `state_dir/events` provides structured interaction data + +4. **Iterate or advance** — if feedback changes current screen, write a new file (e.g., `layout-v2.html`). Only move to the next question when the current step is validated. + +5. **Unload when returning to terminal** — when the next step doesn't need the browser (e.g., a clarifying question, a tradeoff discussion), push a waiting screen to clear the stale content: + + ```html + <!-- filename: waiting.html (or waiting-2.html, etc.) --> + <div style="display:flex;align-items:center;justify-content:center;min-height:60vh"> + <p class="subtitle">Continuing in terminal...</p> + </div> + ``` + + This prevents the user from staring at a resolved choice while the conversation has moved on. When the next visual question comes up, push a new content file as usual. + +6. Repeat until done. + +## Writing Content Fragments + +Write just the content that goes inside the page. The server wraps it in the frame template automatically (header, theme CSS, selection indicator, and all interactive infrastructure). + +**Minimal example:** + +```html +<h2>Which layout works better?</h2> +<p class="subtitle">Consider readability and visual hierarchy</p> + +<div class="options"> + <div class="option" data-choice="a" onclick="toggleSelect(this)"> + <div class="letter">A</div> + <div class="content"> + <h3>Single Column</h3> + <p>Clean, focused reading experience</p> + </div> + </div> + <div class="option" data-choice="b" onclick="toggleSelect(this)"> + <div class="letter">B</div> + <div class="content"> + <h3>Two Column</h3> + <p>Sidebar navigation with main content</p> + </div> + </div> +</div> +``` + +That's it. No `<html>`, no CSS, no `<script>` tags needed. The server provides all of that. + +## CSS Classes Available + +The frame template provides these CSS classes for your content: + +### Options (A/B/C choices) + +```html +<div class="options"> + <div class="option" data-choice="a" onclick="toggleSelect(this)"> + <div class="letter">A</div> + <div class="content"> + <h3>Title</h3> + <p>Description</p> + </div> + </div> +</div> +``` + +**Multi-select:** Add `data-multiselect` to the container to let users select multiple options. Each click toggles the item. The indicator bar shows the count. + +```html +<div class="options" data-multiselect> + <!-- same option markup — users can select/deselect multiple --> +</div> +``` + +### Cards (visual designs) + +```html +<div class="cards"> + <div class="card" data-choice="design1" onclick="toggleSelect(this)"> + <div class="card-image"><!-- mockup content --></div> + <div class="card-body"> + <h3>Name</h3> + <p>Description</p> + </div> + </div> +</div> +``` + +### Mockup container + +```html +<div class="mockup"> + <div class="mockup-header">Preview: Dashboard Layout</div> + <div class="mockup-body"><!-- your mockup HTML --></div> +</div> +``` + +### Split view (side-by-side) + +```html +<div class="split"> + <div class="mockup"><!-- left --></div> + <div class="mockup"><!-- right --></div> +</div> +``` + +### Pros/Cons + +```html +<div class="pros-cons"> + <div class="pros"><h4>Pros</h4><ul><li>Benefit</li></ul></div> + <div class="cons"><h4>Cons</h4><ul><li>Drawback</li></ul></div> +</div> +``` + +### Mock elements (wireframe building blocks) + +```html +<div class="mock-nav">Logo | Home | About | Contact</div> +<div style="display: flex;"> + <div class="mock-sidebar">Navigation</div> + <div class="mock-content">Main content area</div> +</div> +<button class="mock-button">Action Button</button> +<input class="mock-input" placeholder="Input field"> +<div class="placeholder">Placeholder area</div> +``` + +### Typography and sections + +- `h2` — page title +- `h3` — section heading +- `.subtitle` — secondary text below title +- `.section` — content block with bottom margin +- `.label` — small uppercase label text + +## Browser Events Format + +When the user clicks options in the browser, their interactions are recorded to `$STATE_DIR/events` (one JSON object per line). The file is cleared automatically when you push a new screen. + +```jsonl +{"type":"click","choice":"a","text":"Option A - Simple Layout","timestamp":1706000101} +{"type":"click","choice":"c","text":"Option C - Complex Grid","timestamp":1706000108} +{"type":"click","choice":"b","text":"Option B - Hybrid","timestamp":1706000115} +``` + +The full event stream shows the user's exploration path — they may click multiple options before settling. The last `choice` event is typically the final selection, but the pattern of clicks can reveal hesitation or preferences worth asking about. + +If `$STATE_DIR/events` doesn't exist, the user didn't interact with the browser — use only their terminal text. + +## Design Tips + +- **Scale fidelity to the question** — wireframes for layout, polish for polish questions +- **Explain the question on each page** — "Which layout feels more professional?" not just "Pick one" +- **Iterate before advancing** — if feedback changes current screen, write a new version +- **2-4 options max** per screen +- **Use real content when it matters** — for a photography portfolio, use actual images (Unsplash). Placeholder content obscures design issues. +- **Keep mockups simple** — focus on layout and structure, not pixel-perfect design + +## File Naming + +- Use semantic names: `platform.html`, `visual-style.html`, `layout.html` +- Never reuse filenames — each screen must be a new file +- For iterations: append version suffix like `layout-v2.html`, `layout-v3.html` +- Server serves newest file by modification time + +## Cleaning Up + +```bash +scripts/stop-server.sh $SESSION_DIR +``` + +If the session used `--project-dir`, mockup files persist in `.superpowers/brainstorm/` for later reference. Only `/tmp` sessions get deleted on stop. + +## Reference + +- Frame template (CSS reference): `scripts/frame-template.html` +- Helper script (client-side): `scripts/helper.js` \ No newline at end of file diff --git a/.claude/skills/commit/SKILL.md b/.claude/skills/commit/SKILL.md new file mode 100644 index 0000000..998817e --- /dev/null +++ b/.claude/skills/commit/SKILL.md @@ -0,0 +1,68 @@ +--- +name: commit +description: Use when committing code changes - enforces commit message format, safety checks, and prohibits AI signatures +--- + +# Commit + +Automatically generate a commit message and submit the code. + +## Workflow + +1. **Check status**: run `git status` to inspect changes (do not use `-uall`) +2. **Review diff**: run `git diff --staged` and `git diff` to inspect the exact changes +3. **Check style**: run `git log --oneline -10` to review recent commit style +4. **Generate message**: create a commit message that follows the rules below +5. **Commit**: stage the files and commit + +## Commit Message Rules + +### Format +```text +<type>: <description> + +[optional body] +``` + +### Type Values +- `feat`: new feature +- `fix`: bug fix +- `docs`: documentation update +- `refactor`: refactor with no functional change +- `test`: test-related change +- `chore`: build / tooling / configuration +- `perf`: performance optimization +- `ci`: CI/CD related + +### Rules +- The description must be written in English +- Keep it under 72 characters +- Use the imperative mood, for example "add user authentication" instead of "added user authentication" + +## Safety Rules + +> [!CAUTION] +> **Never** add any AI signature to a commit message: +> - Do not add `Co-Authored-By: Claude` +> - Do not add `🤖 Generated with...` +> - Do not add any AI signature or marker + +## Sensitive File Check + +Before committing, check whether any of the following files were accidentally added: +- `.env` / `.env.*` +- `credentials.json` +- `*_secret*` +- `*.pem` / `*.key` + +If sensitive files are found, **warn the user** and wait for confirmation. + +## Examples + +```bash +# Auto-generate a message +/commit + +# Specify a message +/commit "feat: add user login" +``` diff --git a/.claude/skills/finishing-a-development-branch/SKILL.md b/.claude/skills/finishing-a-development-branch/SKILL.md new file mode 100644 index 0000000..c4aa549 --- /dev/null +++ b/.claude/skills/finishing-a-development-branch/SKILL.md @@ -0,0 +1,251 @@ +--- +name: finishing-a-development-branch +description: Use when implementation is complete, all tests pass, and you need to decide how to integrate the work - guides completion of development work by presenting structured options for merge, PR, or cleanup +--- + +# Finishing a Development Branch + +## Overview + +Guide completion of development work by presenting clear options and handling chosen workflow. + +**Core principle:** Verify tests → Detect environment → Present options → Execute choice → Clean up. + +**Announce at start:** "I'm using the finishing-a-development-branch skill to complete this work." + +## The Process + +### Step 1: Verify Tests + +**Before presenting options, verify tests pass:** + +```bash +# Run project's test suite +conda run -n PolyGateway pytest # 本项目;其他栈: npm test / cargo test / go test +``` + +**If tests fail:** +``` +Tests failing (<N> failures). Must fix before completing: + +[Show failures] + +Cannot proceed with merge/PR until tests pass. +``` + +Stop. Don't proceed to Step 2. + +**If tests pass:** Continue to Step 2. + +### Step 2: Detect Environment + +**Determine workspace state before presenting options:** + +```bash +GIT_DIR=$(cd "$(git rev-parse --git-dir)" 2>/dev/null && pwd -P) +GIT_COMMON=$(cd "$(git rev-parse --git-common-dir)" 2>/dev/null && pwd -P) +``` + +This determines which menu to show and how cleanup works: + +| State | Menu | Cleanup | +|-------|------|---------| +| `GIT_DIR == GIT_COMMON` (normal repo) | Standard 4 options | No worktree to clean up | +| `GIT_DIR != GIT_COMMON`, named branch | Standard 4 options | Provenance-based (see Step 6) | +| `GIT_DIR != GIT_COMMON`, detached HEAD | Reduced 3 options (no merge) | No cleanup (externally managed) | + +### Step 3: Determine Base Branch + +```bash +# Try common base branches +git merge-base HEAD main 2>/dev/null || git merge-base HEAD master 2>/dev/null +``` + +Or ask: "This branch split from main - is that correct?" + +### Step 4: Present Options + +**Normal repo and named-branch worktree — present exactly these 4 options:** + +``` +Implementation complete. What would you like to do? + +1. Merge back to <base-branch> locally +2. Push and create a Pull Request +3. Keep the branch as-is (I'll handle it later) +4. Discard this work + +Which option? +``` + +**Detached HEAD — present exactly these 3 options:** + +``` +Implementation complete. You're on a detached HEAD (externally managed workspace). + +1. Push as new branch and create a Pull Request +2. Keep as-is (I'll handle it later) +3. Discard this work + +Which option? +``` + +**Don't add explanation** - keep options concise. + +### Step 5: Execute Choice + +#### Option 1: Merge Locally + +```bash +# Get main repo root for CWD safety +MAIN_ROOT=$(git -C "$(git rev-parse --git-common-dir)/.." rev-parse --show-toplevel) +cd "$MAIN_ROOT" + +# Merge first — verify success before removing anything +git checkout <base-branch> +git pull +git merge <feature-branch> + +# Verify tests on merged result +<test command> + +# Only after merge succeeds: cleanup worktree (Step 6), then delete branch +``` + +Then: Cleanup worktree (Step 6), then delete branch: + +```bash +git branch -d <feature-branch> +``` + +#### Option 2: Push and Create PR + +```bash +# Push branch +git push -u origin <feature-branch> + +# Create PR +gh pr create --title "<title>" --body "$(cat <<'EOF' +## Summary +<2-3 bullets of what changed> + +## Test Plan +- [ ] <verification steps> +EOF +)" +``` + +**Do NOT clean up worktree** — user needs it alive to iterate on PR feedback. + +#### Option 3: Keep As-Is + +Report: "Keeping branch <name>. Worktree preserved at <path>." + +**Don't cleanup worktree.** + +#### Option 4: Discard + +**Confirm first:** +``` +This will permanently delete: +- Branch <name> +- All commits: <commit-list> +- Worktree at <path> + +Type 'discard' to confirm. +``` + +Wait for exact confirmation. + +If confirmed: +```bash +MAIN_ROOT=$(git -C "$(git rev-parse --git-common-dir)/.." rev-parse --show-toplevel) +cd "$MAIN_ROOT" +``` + +Then: Cleanup worktree (Step 6), then force-delete branch: +```bash +git branch -D <feature-branch> +``` + +### Step 6: Cleanup Workspace + +**Only runs for Options 1 and 4.** Options 2 and 3 always preserve the worktree. + +```bash +GIT_DIR=$(cd "$(git rev-parse --git-dir)" 2>/dev/null && pwd -P) +GIT_COMMON=$(cd "$(git rev-parse --git-common-dir)" 2>/dev/null && pwd -P) +WORKTREE_PATH=$(git rev-parse --show-toplevel) +``` + +**If `GIT_DIR == GIT_COMMON`:** Normal repo, no worktree to clean up. Done. + +**If worktree path is under `.worktrees/`, `worktrees/`, or `~/.config/superpowers/worktrees/`:** Superpowers created this worktree — we own cleanup. + +```bash +MAIN_ROOT=$(git -C "$(git rev-parse --git-common-dir)/.." rev-parse --show-toplevel) +cd "$MAIN_ROOT" +git worktree remove "$WORKTREE_PATH" +git worktree prune # Self-healing: clean up any stale registrations +``` + +**Otherwise:** The host environment (harness) owns this workspace. Do NOT remove it. If your platform provides a workspace-exit tool, use it. Otherwise, leave the workspace in place. + +## Quick Reference + +| Option | Merge | Push | Keep Worktree | Cleanup Branch | +|--------|-------|------|---------------|----------------| +| 1. Merge locally | yes | - | - | yes | +| 2. Create PR | - | yes | yes | - | +| 3. Keep as-is | - | - | yes | - | +| 4. Discard | - | - | - | yes (force) | + +## Common Mistakes + +**Skipping test verification** +- **Problem:** Merge broken code, create failing PR +- **Fix:** Always verify tests before offering options + +**Open-ended questions** +- **Problem:** "What should I do next?" is ambiguous +- **Fix:** Present exactly 4 structured options (or 3 for detached HEAD) + +**Cleaning up worktree for Option 2** +- **Problem:** Remove worktree user needs for PR iteration +- **Fix:** Only cleanup for Options 1 and 4 + +**Deleting branch before removing worktree** +- **Problem:** `git branch -d` fails because worktree still references the branch +- **Fix:** Merge first, remove worktree, then delete branch + +**Running git worktree remove from inside the worktree** +- **Problem:** Command fails silently when CWD is inside the worktree being removed +- **Fix:** Always `cd` to main repo root before `git worktree remove` + +**Cleaning up harness-owned worktrees** +- **Problem:** Removing a worktree the harness created causes phantom state +- **Fix:** Only clean up worktrees under `.worktrees/`, `worktrees/`, or `~/.config/superpowers/worktrees/` + +**No confirmation for discard** +- **Problem:** Accidentally delete work +- **Fix:** Require typed "discard" confirmation + +## Red Flags + +**Never:** +- Proceed with failing tests +- Merge without verifying tests on result +- Delete work without confirmation +- Force-push without explicit request +- Remove a worktree before confirming merge success +- Clean up worktrees you didn't create (provenance check) +- Run `git worktree remove` from inside the worktree + +**Always:** +- Verify tests before offering options +- Detect environment before presenting menu +- Present exactly 4 options (or 3 for detached HEAD) +- Get typed confirmation for Option 4 +- Clean up worktree for Options 1 & 4 only +- `cd` to main repo root before worktree removal +- Run `git worktree prune` after removal diff --git a/.claude/skills/graphify/SKILL.md b/.claude/skills/graphify/SKILL.md new file mode 100644 index 0000000..efde04e --- /dev/null +++ b/.claude/skills/graphify/SKILL.md @@ -0,0 +1,616 @@ +--- +name: graphify +description: "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." +trigger: /graphify +--- + +# /graphify + +Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. + +## Usage + +``` +/graphify # full pipeline on current directory → Obsidian vault +/graphify <path> # full pipeline on specific path +/graphify https://github.com/<owner>/<repo> # clone repo then run full pipeline on it +/graphify https://github.com/<owner>/<repo> --branch <branch> # clone a specific branch +/graphify <url1> <url2> ... # clone multiple repos, build each, merge into one cross-repo graph +/graphify <path> --mode deep # thorough extraction, richer INFERRED edges +/graphify <path> --update # incremental - re-extract only new/changed files +/graphify <path> --directed # build directed graph (preserves edge direction: source→target) +/graphify <path> --whisper-model medium # use a larger Whisper model for better transcription accuracy +/graphify <path> --cluster-only # rerun clustering on existing graph +/graphify <path> --no-viz # skip visualization, just report + JSON +/graphify <path> --html # (HTML is generated by default - this flag is a no-op) +/graphify <path> --svg # also export graph.svg (embeds in Notion, GitHub) +/graphify <path> --graphml # export graph.graphml (Gephi, yEd) +/graphify <path> --neo4j # generate graphify-out/cypher.txt for Neo4j +/graphify <path> --neo4j-push bolt://localhost:7687 # push directly to Neo4j +/graphify <path> --mcp # start MCP stdio server for agent access +/graphify <path> --watch # watch folder, auto-rebuild on code changes (no LLM needed) +/graphify <path> --wiki # build agent-crawlable wiki (index.md + one article per community) +/graphify <path> --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) +/graphify add <url> # fetch URL, save to ./raw, update graph +/graphify add <url> --author "Name" # tag who wrote it +/graphify add <url> --contributor "Name" # tag who added it to the corpus +/graphify query "<question>" # BFS traversal - broad context +/graphify query "<question>" --dfs # DFS - trace a specific path +/graphify query "<question>" --budget 1500 # cap answer at N tokens +/graphify path "AuthModule" "Database" # shortest path between two concepts +/graphify explain "SwinTransformer" # plain-language explanation of a node +``` + +## What graphify is for + +Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. + +## What You Must Do When Invoked + +If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. + +**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query "<question>"` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. + +If no path was given, use `.` (current directory). Do not ask the user for a path. + +If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. + +Follow these steps in order. Do not skip steps. + +### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) + +Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. + +### Step 1 - Ensure graphify is installed + +```bash +# Detect the correct Python interpreter (handles uv tool, pipx, venv, system installs) +PYTHON="" +GRAPHIFY_BIN=$(which graphify 2>/dev/null) +# 1. uv tool installs — most reliable on modern Mac/Linux +if [ -z "$PYTHON" ] && command -v uv >/dev/null 2>&1; then + _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi +fi +# 2. Read shebang from graphify binary (pipx and direct pip installs) +if [ -z "$PYTHON" ] && [ -n "$GRAPHIFY_BIN" ]; then + _SHEBANG=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$_SHEBANG" in + *[!a-zA-Z0-9/_.-]*) ;; + *) "$_SHEBANG" -c "import graphify" 2>/dev/null && PYTHON="$_SHEBANG" ;; + esac +fi +# 3. Fall back to python3 +if [ -z "$PYTHON" ]; then PYTHON="python3"; fi +if ! "$PYTHON" -c "import graphify" 2>/dev/null; then + if command -v uv >/dev/null 2>&1; then + uv tool install --upgrade graphifyy -q 2>&1 | tail -3 + _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi + else + "$PYTHON" -m pip install graphifyy -q 2>/dev/null \ + || "$PYTHON" -m pip install graphifyy -q --break-system-packages 2>&1 | tail -3 + fi +fi +# Write interpreter path for all subsequent steps (persists across invocations) +mkdir -p graphify-out +"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" +# Save scan root so `graphify update` (no args) knows where to look next time +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root +``` + +If the import succeeds, print nothing and move straight to Step 2. + +**In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** + +### Step 2 - Detect files + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.detect import detect +from pathlib import Path +result = detect(Path('INPUT_PATH')) +print(json.dumps(result, ensure_ascii=False)) +" > graphify-out/.graphify_detect.json +``` + +Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: + +``` +Corpus: X files · ~Y words + code: N files (.py .ts .go ...) + docs: N files (.md .txt ...) + papers: N files (.pdf ...) + images: N files + video: N files (.mp4 .mp3 ...) +``` + +Omit any category with 0 files from the summary. + +Then act on it: +- If `total_files` is 0: stop with "No supported files found in [path]." +- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. +- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: + - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). + - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). + - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. + - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. + - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. + - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. +- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. + +### Step 2.5 - Video and audio (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. + +### Step 3 - Extract entities and relationships + +**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. + +This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). + +**Before dispatching subagents:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: +> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). + +Print it once, then continue. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching Claude subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. + +> **No other API keys are read.** If `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, fall straight through to Claude Code subagent dispatch (Part B below) — the host session itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key from the environment. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. + +**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** + +Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. + +#### Part A - Structural extraction for code files + +For any code files detected, run AST extraction in parallel with Part B subagents: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.extract import collect_files, extract +from pathlib import Path +import json + +code_files = [] +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +for f in detect.get('files', {}).get('code', []): + code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) + +if code_files: + result = extract(code_files, cache_root=Path('.')) + Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") + print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') +else: + Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") + print('No code files - skipping AST extraction') +" +``` + +#### Part B - Semantic extraction (parallel subagents) + +**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. + +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** + +Before dispatching subagents, print a timing estimate: +- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` +- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) +- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) +- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" + +**Step B0 - Check extraction cache first** + +Before dispatching any subagents, check which files already have cached extraction results: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.cache import check_semantic_cache +from pathlib import Path + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +all_files = [f for files in detect['files'].values() for f in files] + +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) + +if cached_nodes or cached_edges or cached_hyperedges: + Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") +print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') +" +``` + +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. + +**Step B1 - Split into chunks** + +Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. + +**Step B2 - Dispatch ALL subagents in a single message** + +Call the Agent tool multiple times IN THE SAME RESPONSE - one call per chunk. This is the only way they run in parallel. If you make one Agent call, wait, then make another, you are doing it sequentially and defeating the purpose. + +**IMPORTANT - subagent type:** Always use `subagent_type="general-purpose"`. Do NOT use `Explore` - it is read-only and cannot write chunk files to disk, which silently drops extraction results. General-purpose has Write and Bash access which the subagent needs. + +Concrete example for 3 chunks: +``` +[Agent tool call 1: files 1-15, subagent_type="general-purpose"] +[Agent tool call 2: files 16-30, subagent_type="general-purpose"] +[Agent tool call 3: files 31-45, subagent_type="general-purpose"] +``` +All three in one message. Not three separate messages. + +Each subagent receives this exact prompt (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). + +CHUNK_PATH must be an **absolute** path — derive it before dispatching: +```bash +PROJECT_ROOT=$(cat graphify-out/.graphify_root) +# Then for chunk N: CHUNK_PATH="${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" +``` + +Subagent prompt template: + +See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, frontmatter, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. + +**Step B3 - Collect, cache, and merge** + +Wait for all subagents. For each result: +- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal +- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache +- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. +- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort + +If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. + +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +```bash +$(cat graphify-out/.graphify_python) -c " +import json, glob +from pathlib import Path + +chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +all_nodes, all_edges, all_hyperedges = [], [], [] +total_in, total_out = 0, 0 +for c in chunks: + d = json.loads(Path(c).read_text(encoding=\"utf-8\")) + all_nodes += d.get('nodes', []) + all_edges += d.get('edges', []) + all_hyperedges += d.get('hyperedges', []) + total_in += d.get('input_tokens', 0) + total_out += d.get('output_tokens', 0) +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ + 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, + 'input_tokens': total_in, 'output_tokens': total_out, +}, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +" +``` + +Save new results to cache: +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.cache import save_semantic_cache +from pathlib import Path + +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) +print(f'Cached {saved} files') +" +``` + +Merge cached + new results into `graphify-out/.graphify_semantic.json`: +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} + +all_nodes = cached['nodes'] + new.get('nodes', []) +all_edges = cached['edges'] + new.get('edges', []) +all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) +seen = set() +deduped = [] +for n in all_nodes: + if n['id'] not in seen: + seen.add(n['id']) + deduped.append(n) + +merged = { + 'nodes': deduped, + 'edges': all_edges, + 'hyperedges': all_hyperedges, + 'input_tokens': new.get('input_tokens', 0), + 'output_tokens': new.get('output_tokens', 0), +} +Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') +" +``` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` + +#### Part C - Merge AST + semantic into final extraction + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from pathlib import Path + +ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) +sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) + +# Merge: AST nodes first, semantic nodes deduplicated by id +seen = {n['id'] for n in ast['nodes']} +merged_nodes = list(ast['nodes']) +for n in sem['nodes']: + if n['id'] not in seen: + merged_nodes.append(n) + seen.add(n['id']) + +merged_edges = ast['edges'] + sem['edges'] +merged_hyperedges = sem.get('hyperedges', []) +merged = { + 'nodes': merged_nodes, + 'edges': merged_edges, + 'hyperedges': merged_hyperedges, + 'input_tokens': sem.get('input_tokens', 0), + 'output_tokens': sem.get('output_tokens', 0), +} +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") +total = len(merged_nodes) +edges = len(merged_edges) +print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') +" +``` + +### Step 4 - Build graph, cluster, analyze, generate outputs + +**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. + +```bash +mkdir -p graphify-out +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import cluster, score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from graphify.export import to_json +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) + +G = build_from_json(extraction) +communities = cluster(G) +cohesion = score_all(G, communities) +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} +gods = god_nodes(G) +surprises = surprising_connections(G, communities) +labels = {cid: 'Community ' + str(cid) for cid in communities} +# Placeholder questions - regenerated with real labels in Step 5 +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +to_json(G, communities, 'graphify-out/graph.json') + +analysis = { + 'communities': {str(k): v for k, v in communities.items()}, + 'cohesion': {str(k): v for k, v in cohesion.items()}, + 'gods': gods, + 'surprises': surprises, + 'questions': questions, +} +Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") +if G.number_of_nodes() == 0: + print('ERROR: Graph is empty - extraction produced no nodes.') + print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') + raise SystemExit(1) +print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') +" +``` + +If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. + +Replace INPUT_PATH with the actual path. + +### Step 5 - Label communities + +Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). + +Then regenerate the report and save the labels for the visualizer: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +cohesion = {int(k): v for k, v in analysis['cohesion'].items()} +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} + +# LABELS - replace these with the names you chose above +labels = LABELS_DICT + +# Regenerate questions with real community labels (labels affect question phrasing) +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") +print('Report updated with community labels') +" +``` + +Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). +Replace INPUT_PATH with the actual path. + +### Step 6 - Generate Obsidian vault (opt-in) + HTML + +**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. + +If `--obsidian` was given: + +- If `--obsidian-dir <path>` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. + +```bash +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project +``` + +Generate the HTML graph (always, unless `--no-viz`): + +```bash +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz +``` + +### Steps 6b-8 - Wiki, Neo4j, SVG, GraphML, MCP, benchmark (only on their flags) + +These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. + +--- + +### Step 9 - Save manifest, update cost tracker, clean up, and report + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from datetime import datetime, timezone +from graphify.detect import save_manifest + +# Save manifest for --update +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +# In --update mode, 'all_files' carries the full corpus; 'files' is the changed +# subset. Full-rebuild mode populates only 'files', so the fallback handles that. +save_manifest(detect.get('all_files') or detect['files']) + +# Update cumulative cost tracker +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +input_tok = extract.get('input_tokens', 0) +output_tok = extract.get('output_tokens', 0) + +cost_path = Path('graphify-out/cost.json') +if cost_path.exists(): + cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) +else: + cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} + +cost['runs'].append({ + 'date': datetime.now(timezone.utc).isoformat(), + 'input_tokens': input_tok, + 'output_tokens': output_tok, + 'files': detect.get('total_files', 0), +}) +cost['total_input_tokens'] += input_tok +cost['total_output_tokens'] += output_tok +cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") + +print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') +print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') +" +rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json +find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null +rm -f graphify-out/.needs_update 2>/dev/null || true +``` + +Tell the user (omit the obsidian line unless --obsidian was given): +``` +Graph complete. Outputs in PATH_TO_DIR/graphify-out/ + + graph.html - interactive graph, open in browser + GRAPH_REPORT.md - audit report + graph.json - raw graph data + obsidian/ - Obsidian vault (only if --obsidian was given) +``` + +If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi + +Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. + +Then paste these sections from GRAPH_REPORT.md directly into the chat: +- God Nodes +- Surprising Connections +- Suggested Questions + +Do NOT paste the full report - just those three sections. Keep it concise. + +Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: + +> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" + +If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. + +The graph is the map. Your job after the pipeline is to be the guide. + +--- + +## Interpreter guard for subcommands + +Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: + +```bash +if [ ! -f graphify-out/.graphify_python ]; then + GRAPHIFY_BIN=$(which graphify 2>/dev/null) + if [ -n "$GRAPHIFY_BIN" ]; then + PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$PYTHON" in *[!a-zA-Z0-9/_.-]*) PYTHON="python3" ;; esac + else + PYTHON="python3" + fi + mkdir -p graphify-out + "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" +fi +``` + +## For --update and --cluster-only + +Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. + +--- + +## For /graphify query + +When `graphify-out/graph.json` already exists and the user asks a question about the corpus, run the query directly: + +```bash +graphify query "<question>" +``` + +Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. For that vocab-expansion step, the `--dfs` / `--budget` modes, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. + +--- + +## For /graphify add and --watch + +Neither is part of the default build. When the user runs `/graphify add <url>` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. + +--- + +## For the commit hook and native CLAUDE.md integration + +When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. + +--- + +## Honesty Rules + +- Never invent an edge. If unsure, use AMBIGUOUS. +- Never skip the corpus check warning. +- Always show token cost in the report. +- Never hide cohesion scores behind symbols - show the raw number. +- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. diff --git a/.claude/skills/graphify/references/add-watch.md b/.claude/skills/graphify/references/add-watch.md new file mode 100644 index 0000000..78b6870 --- /dev/null +++ b/.claude/skills/graphify/references/add-watch.md @@ -0,0 +1,56 @@ +# graphify reference: add a URL and watch a folder + +Load this when the user ran `/graphify add <url>` or passed `--watch`. Neither is part of the default build. + +## For /graphify add + +Fetch a URL and add it to the corpus, then update the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys +from graphify.ingest import ingest +from pathlib import Path + +try: + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') + print(f'Saved to {out}') +except ValueError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +except RuntimeError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +" +``` + +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. + +Supported URL types (auto-detected): +- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) +- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author +- arXiv → abstract + metadata saved as `.md` +- PDF → downloaded as `.pdf` +- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run +- Any webpage → converted to markdown via html2text + +--- + +## For --watch + +Start a background watcher that monitors a folder and auto-updates the graph when files change. + +```bash +python3 -m graphify.watch INPUT_PATH --debounce 3 +``` + +Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: + +- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. +- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). + +Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. + +Press Ctrl+C to stop. + +For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. diff --git a/.claude/skills/graphify/references/exports.md b/.claude/skills/graphify/references/exports.md new file mode 100644 index 0000000..3750ff2 --- /dev/null +++ b/.claude/skills/graphify/references/exports.md @@ -0,0 +1,71 @@ +# graphify reference: extra exports and benchmark + +Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. + +### Step 6b - Wiki (only if --wiki flag) + +**Only run this step if `--wiki` was explicitly given in the original command.** + +Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. + +```bash +graphify export wiki +``` + +### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) + +**If `--neo4j`** - generate a Cypher file for manual import: + +```bash +graphify export neo4j +``` + +**If `--neo4j-push <uri>`** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: + +```bash +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD +``` + +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. + +### Step 7b - SVG export (only if --svg flag) + +```bash +graphify export svg +``` + +### Step 7c - GraphML export (only if --graphml flag) + +```bash +graphify export graphml +``` + +### Step 7d - MCP server (only if --mcp flag) + +```bash +python3 -m graphify.serve graphify-out/graph.json +``` + +This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. + +To configure in Claude Desktop, add to `claude_desktop_config.json`: +```json +{ + "mcpServers": { + "graphify": { + "command": "python3", + "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] + } + } +} +``` + +### Step 8 - Token reduction benchmark (only if total_words > 5000) + +If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: + +```bash +graphify benchmark +``` + +Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. diff --git a/.claude/skills/graphify/references/extraction-spec.md b/.claude/skills/graphify/references/extraction-spec.md new file mode 100644 index 0000000..2bfb873 --- /dev/null +++ b/.claude/skills/graphify/references/extraction-spec.md @@ -0,0 +1,68 @@ +# graphify reference: extraction subagent prompt + +Load this in Step 3 Part B when the corpus has at least one doc, paper, or image chunk. A pure-code corpus skips Part B and never reads this file. Each semantic subagent receives the prompt below verbatim (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). + +``` +You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. +Output ONLY valid JSON matching the schema below - no explanation, no markdown fences, no preamble. + +Files (chunk CHUNK_NUM of TOTAL_CHUNKS): +FILE_LIST + +Rules: +- EXTRACTED: relationship explicit in source (import, call, citation, "see §3.2") +- INFERRED: reasonable inference (shared data structure, implied dependency) +- AMBIGUOUS: uncertain - flag for review, do not omit + +Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). + Do not re-extract imports - AST already has those. +Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. `calls` edges MUST stay within one language: a Python function cannot `calls` a JS/TS/Go/Rust/Java symbol and vice versa — cross-language call edges are phantom artifacts, never emit them. +Image files: use vision to understand what the image IS - do not just OCR. + UI screenshot: layout patterns, design decisions, key elements, purpose. + Chart: metric, trend/insight, data source. + Tweet/post: claim as node, author, concepts mentioned. + Diagram: components and connections. + Research figure: what it demonstrates, method, result. + Handwritten/whiteboard: ideas and arrows, mark uncertain readings AMBIGUOUS. + +DEEP_MODE (if --mode deep was given): be aggressive with INFERRED edges - indirect deps, + shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. + +Semantic similarity: if two concepts in this chunk solve the same problem or represent the same idea without any structural link (no import, no call, no citation), add a `semantically_similar_to` edge marked INFERRED with a confidence_score reflecting how similar they are (0.6-0.95). Examples: +- Two functions that both validate user input but never call each other +- A class in code and a concept in a paper that describe the same algorithm +- Two error types that handle the same failure mode differently +Only add these when the similarity is genuinely non-obvious and cross-cutting. Do not add them for trivially similar things. + +Hyperedges: if 3 or more nodes clearly participate together in a shared concept, flow, or pattern that is not captured by pairwise edges alone, add a hyperedge to a top-level `hyperedges` array. Examples: +- All classes that implement a common protocol or interface +- All functions in an authentication flow (even if they don't all call each other) +- All concepts from a paper section that form one coherent idea +Use sparingly — only when the group relationship adds information beyond the pairwise edges. Maximum 3 hyperedges per chunk. + +If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, + contributor onto every node from that file. + +confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a default: +- EXTRACTED edges: confidence_score = 1.0 always +- INFERRED edges: pick exactly ONE value from this set — never 0.5: + 0.95 direct structural evidence (shared data structure, named cross-file reference). + 0.85 strong inference (clear functional alignment, no direct symbol link). + 0.75 reasonable inference (shared problem domain + similar shape, requires interpretation). + 0.65 weak inference (thematically related, no shape evidence). + 0.55 speculative but plausible (surface-level co-occurrence only). + Models follow discrete rubrics better than continuous ranges; the bimodal + distribution observed in production (>50% at 0.5, >40% at 0.85+) shows the + range guidance is being collapsed to a binary. If no value above fits, mark + the edge AMBIGUOUS rather than picking 0.4 or below. +- AMBIGUOUS edges: 0.1-0.3 + +Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the **immediate** parent directory name + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent is used — not the full path. Examples: `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or the full path (e.g., `src_auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project that had ghost duplicates under the old format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. + +Generate the extraction JSON matching this schema exactly: +{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} + +Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): +CHUNK_PATH +``` diff --git a/.claude/skills/graphify/references/github-and-merge.md b/.claude/skills/graphify/references/github-and-merge.md new file mode 100644 index 0000000..a41ea06 --- /dev/null +++ b/.claude/skills/graphify/references/github-and-merge.md @@ -0,0 +1,46 @@ +# graphify reference: GitHub clone and cross-repo merge + +Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. + +### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) + +**Single repo:** +```bash +LOCAL_PATH=$(graphify clone <github-url> [--branch <branch>]) +# Use LOCAL_PATH as the target for all subsequent steps +``` + +**Multiple repos (cross-repo graph):** +```bash +# Clone each repo, run the full pipeline on each, then merge +graphify clone <url1> # → ~/.graphify/repos/<owner1>/<repo1> +graphify clone <url2> # → ~/.graphify/repos/<owner2>/<repo2> +# Run /graphify on each local path to produce their graph.json files +# Then merge: +graphify merge-graphs \ + ~/.graphify/repos/<owner1>/<repo1>/graphify-out/graph.json \ + ~/.graphify/repos/<owner2>/<repo2>/graphify-out/graph.json \ + --out graphify-out/cross-repo-graph.json +``` + +Graphify clones into `~/.graphify/repos/<owner>/<repo>` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. + +**Multiple local subfolders (monorepo or multi-service layout):** + +The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: + +```bash +graphify extract ./core/ # → ./core/graphify-out/graph.json +graphify extract ./service/ # → ./service/graphify-out/graph.json +graphify extract ./platform/ # → ./platform/graphify-out/graph.json +# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set + +# Then merge at the project root: +graphify merge-graphs \ + ./core/graphify-out/graph.json \ + ./service/graphify-out/graph.json \ + ./platform/graphify-out/graph.json \ + --out graphify-out/graph.json +``` + +Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. diff --git a/.claude/skills/graphify/references/hooks.md b/.claude/skills/graphify/references/hooks.md new file mode 100644 index 0000000..438b8b1 --- /dev/null +++ b/.claude/skills/graphify/references/hooks.md @@ -0,0 +1,33 @@ +# graphify reference: commit hook and native CLAUDE.md integration + +Load this when the user asked to install the post-commit hook or wire graphify into a project's CLAUDE.md. + +## For git commit hook + +Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. + +```bash +graphify hook install # install +graphify hook uninstall # remove +graphify hook status # check +``` + +After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. + +If a post-commit hook already exists, graphify appends to it rather than replacing it. + +--- + +## For native CLAUDE.md integration + +Run once per project to make graphify always-on in Claude Code sessions: + +```bash +graphify claude install +``` + +This writes a `## graphify` section to the local `CLAUDE.md` that instructs Claude to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. + +```bash +graphify claude uninstall # remove the section +``` diff --git a/.claude/skills/graphify/references/query.md b/.claude/skills/graphify/references/query.md new file mode 100644 index 0000000..b08289e --- /dev/null +++ b/.claude/skills/graphify/references/query.md @@ -0,0 +1,103 @@ +# graphify reference: query, path, explain + +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. + +Two traversal modes - choose based on the question: + +| Mode | Flag | Best for | +|------|------|----------| +| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | +| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | + +### Step 0 — Constrained query expansion (REQUIRED before traversal) + +graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. + +Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: + +1. Extract the token vocabulary from node labels: +```bash +$(cat graphify-out/.graphify_python) -c " +import json, re +from pathlib import Path +data = json.loads(Path('graphify-out/graph.json').read_text()) +vocab = set() +for n in data['nodes']: + for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): + parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] + for p in parts: + t = p.lower() + if 3 <= len(t) <= 30: + vocab.add(t) +Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab))) +print(f'vocab: {len(vocab)} tokens') +" +``` + +2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: + - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. + - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. + - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. + - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. + - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. + +3. Print the selection explicitly to the user before running the query, so the expansion is auditable: +``` +Query expanded to (from graph vocab, N tokens): [token1, token2, ...] +``` +If the list is empty, say so plainly and stop — do not proceed to traversal. + +### Step 1 — Traversal + +Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) + +```bash +graphify query "QUESTION" +# or: graphify query "QUESTION" --dfs --budget 3000 +``` + +Answer using **only** what the graph output contains. Quote `source_location` when citing a specific fact. If the graph lacks enough information, say so - do not hallucinate edges. + +After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +``` + +Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. + +--- + +## For /graphify path + +Find the shortest path between two named concepts in the graph. + +```bash +graphify path "NODE_A" "NODE_B" +``` + +Replace `NODE_A` and `NODE_B` with the actual concept names. Then explain the path in plain language - what each hop means, why it's significant. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B +``` + +--- + +## For /graphify explain + +Give a plain-language explanation of a single node - everything connected to it. + +```bash +graphify explain "NODE_NAME" +``` + +Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME +``` diff --git a/.claude/skills/graphify/references/transcribe.md b/.claude/skills/graphify/references/transcribe.md new file mode 100644 index 0000000..d9cc698 --- /dev/null +++ b/.claude/skills/graphify/references/transcribe.md @@ -0,0 +1,48 @@ +# graphify reference: transcribe video and audio + +Load this only when `detect` reported one or more `video` files. A corpus with no video never reads this. + +### Step 2.5 - Transcribe video / audio files (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. + +Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. + +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` + +**Step 1 - Write the Whisper prompt yourself.** + +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: + +- Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` +- Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` + +Set it as `WHISPER_PROMPT` to use in the next command. + +**Step 2 - Transcribe:** + +```bash +GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +$(cat graphify-out/.graphify_python) -c " +import json, os +from pathlib import Path +from graphify.transcribe import transcribe_all + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +video_files = detect.get('files', {}).get('video', []) +prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') + +transcript_paths = transcribe_all(video_files, initial_prompt=prompt) +print(json.dumps(transcript_paths, ensure_ascii=False)) +" > graphify-out/.graphify_transcripts.json +``` + +After transcription: +- Read the transcript paths from `graphify-out/.graphify_transcripts.json` +- Add them to the docs list before dispatching semantic subagents in Step 3B +- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` +- If transcription fails for a file, print a warning and continue with the rest + +**Whisper model:** Default is `base`. If the user passed `--whisper-model <name>`, set `GRAPHIFY_WHISPER_MODEL=<name>` in the environment before running the command above. diff --git a/.claude/skills/graphify/references/update.md b/.claude/skills/graphify/references/update.md new file mode 100644 index 0000000..d35b665 --- /dev/null +++ b/.claude/skills/graphify/references/update.md @@ -0,0 +1,179 @@ +# graphify reference: incremental update and cluster-only + +Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. + +## For --update (incremental re-extraction) + +Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.detect import detect_incremental, save_manifest +from pathlib import Path + +result = detect_incremental(Path('INPUT_PATH')) +new_total = result.get('new_total', 0) +print(json.dumps(result, indent=2, ensure_ascii=False)) +Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") +deleted = list(result.get('deleted_files', [])) +if new_total == 0 and not deleted: + print('No files changed since last run. Nothing to update.') + raise SystemExit(0) +if deleted: + print(f'{len(deleted)} deleted file(s) to prune.') +if new_total > 0: + print(f'{new_total} new/changed file(s) to re-extract.') +" +``` + +Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ + 'files': r.get('new_files', {}), + 'all_files': r.get('files', {}), + 'total_files': r.get('new_total', 0), + 'total_words': r.get('total_words', 0), + 'skipped_sensitive': r.get('skipped_sensitive', []), + 'needs_graph': True, +}, ensure_ascii=False), encoding=\"utf-8\") +" +``` + +If new files exist, first check whether all changed files are code files: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} +code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} +new_files = result.get('new_files', {}) +all_changed = [f for files in new_files.values() for f in files] +code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) +print('code_only:', code_only) +" +``` + +If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. + +If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. + + +If no new files exist (only deletions), create an empty extraction so the merge step can prune: + +```bash +if [ ! -f graphify-out/.graphify_extract.json ]; then + echo '[graphify update] Only deletions -- creating empty extraction for merge.' + $(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') +" +fi +``` + + +Then: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from graphify.build import build_merge +from graphify.detect import save_manifest + +# Load new extraction and incremental state +new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +deleted = list(incremental.get('deleted_files', [])) +# Also prune old nodes for re-extracted (changed) files before inserting fresh AST. +# Without this, build_merge's dedup pass tries to reconcile old and new versions of +# the same file's nodes and can collapse same-named symbols across files (#1178). +changed = [f for files in incremental.get('new_files', {}).values() for f in files] +prune = list(dict.fromkeys(deleted + changed)) or None + +# Use build_merge() — reads graph.json directly without NetworkX round-trip +# so edge direction (calls, implements, imports) is always preserved (#801). +G = build_merge( + [new_extraction], + graph_path='graphify-out/graph.json', + prune_sources=prune, +) +print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') + +# Write merged result back to .graphify_extract.json so Step 4 sees the full graph +merged_out = { + 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], + 'edges': [ + # Explicit source/target last so they win over any stale attrs in d. + {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, + 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + for u, v, d in G.edges(data=True) + ], + # G.graph["hyperedges"] holds hyperedges from both existing graph.json + # and new_extraction (build_merge combines them). Falling back to + # new_extraction only would silently drop prior-run hyperedges (#801). + 'hyperedges': list(G.graph.get('hyperedges', [])), + 'input_tokens': new_extraction.get('input_tokens', 0), + 'output_tokens': new_extraction.get('output_tokens', 0), +} +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") +print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') + +# Save manifest so next --update diffs against today's state, not the +# prior run's baseline (prevents ghost-node reports on subsequent updates). +save_manifest(incremental['files']) +print('[graphify update] Manifest saved.') +" +``` + +Then run Steps 4–8 on the merged graph as normal. + +After Step 4, show the graph diff: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.analyze import graph_diff +from graphify.build import build_from_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +# Load old graph (before update) from backup written before merge +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None +new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +G_new = build_from_json(new_extract) + +if old_data: + G_old = json_graph.node_link_graph(old_data, edges='links') + diff = graph_diff(G_old, G_new) + print(diff['summary']) + if diff['new_nodes']: + print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) + if diff['new_edges']: + print('New edges:', len(diff['new_edges'])) +" +``` + +Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` +Clean up after: `rm -f graphify-out/.graphify_old.json` + +--- + +## For --cluster-only + +Skip Steps 1–3. Re-run clustering on the existing graph: + +```bash +graphify cluster-only . +``` + +Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). diff --git a/.claude/skills/harness-eval/SKILL.md b/.claude/skills/harness-eval/SKILL.md new file mode 100644 index 0000000..a2e5587 --- /dev/null +++ b/.claude/skills/harness-eval/SKILL.md @@ -0,0 +1,130 @@ +--- +name: harness-eval +description: "评估功能的真实运行性能。从 Wiki 读取 schema 和 metric 定义，查询 SQLite 日志，对比基线，调用 LLM judge 做语义评估，产出诊断和迭代建议。触发短语: harness eval, 评估性能, 性能评估。" +argument-hint: "[run_id]" +--- + +# Harness Eval + +## Overview + +在真实环境中运行程序后，评估性能是否达标，诊断问题，决定是否迭代。 + +**核心原则：** 不是"通过就停"，而是基于真实指标判断"跑得好不好"。 + +## 触发方式 + +1. **自动**：subagent-driven-development Step 10，所有任务完成 + final review 通过后 +2. **手动**：`/harness-eval` 或 `/harness-eval <run_id>` + +## 前置条件 + +- `research-wiki/schemas/` 中有至少一个 schema 实体 +- `research-wiki/metrics/` 中有至少一个 metric 实体 +- `results/harness.db` 存在且包含数据 +- `.env` 中有 LLM API key（用于 LlmJudge） + +## 长时命令执行约定 + +对真实样本重跑流水线属于长时操作。超过 30s 的命令建议在 tmux 中后台运行，便于人工随时 attach 检查进度；短命令直接前台执行即可。 + +## 执行流程 + +### Phase 1: 准备 + +1. 列出 `research-wiki/schemas/` 下所有 schema 实体，获取表名和列信息 +2. 列出 `research-wiki/metrics/` 下所有 metric 实体，获取基线值和阈值 +3. 确认 `results/harness.db` 存在，查询 `_runs` 表获取最新运行状态 +4. 如果指定了 run_id，使用该 run_id；否则使用最近一次 status='completed' 的 run + +### Phase 2: 运行 + +#### 2a: 检查已有运行记录 + +在 subagent-driven-development 流程中，某个子任务可能已经执行过全局运行并将结果写入了 `harness.db`。此时 harness-eval **不应重复运行**，直接复用已有记录即可。 + +1. 查询 `_runs` 表，查找 `status='completed'` 且 `git_sha` 与当前 `HEAD` 匹配的记录 +2. **找到匹配记录** → 使用该记录的 run_id，**跳过运行**，直接进入 Phase 3。在最终报告中注明「复用已有运行: run_id=xxx」 +3. **未找到匹配记录 + 自动触发** → 继续 2b +4. **未找到匹配记录 + 手动触发** → 询问用户是否需要执行运行 + +#### 2b: 重跑流水线（仅无可用记录时） + +1. `make deploy-testing` 起服务（api / worker / Postgres / Redis / 对象存储） +2. 对真实样本重跑流水线，结果落 `results/harness.db`；属长时操作，建议按「长时命令执行约定」在 tmux 中运行 +3. 轮询检查 `_runs` 表状态，等待完成 +4. 确认 `_runs` 表中对应 run 的 status = 'completed' +5. 如果 status = 'failed'，直接进入 Phase 6 诊断 + +### Phase 3: 硬性指标检查 + +对 `research-wiki/metrics/` 中每个标记为硬性判定的指标： + +1. 从 metric 实体中读取：指标名、基线值、阈值、对应的 schema（表名和列名） +2. 执行 SQL 查询获取当前运行的实际值 +3. 对比：实际值 vs 阈值 +4. 记录结果：pass / fail + 具体数值 + +### Phase 4: 语义评估 + +对 `research-wiki/metrics/` 中每个标记为语义判定的指标： + +1. 收集 evidence：相关表的数据、_events 表的事件、配置信息 +2. 调用 `LlmJudge.evaluate(EvalRequest)`（请求体封装 criteria、evidence、rubric） +3. 记录结果：Verdict + +### Phase 5: 综合判定 + +- 硬性指标全部 pass **且** 语义评估全部 pass → **通过** +- 任一不通过 → **不通过**，进入 Phase 6 + +### Phase 6: 诊断与迭代决策（仅不通过时） + +1. 收集失败信息：哪些指标不达标、实际值与期望值的差距 +2. 查询 `_events` 表获取运行过程中的关键事件 +3. 调用 `LlmJudge.diagnose(error_context, log)` +4. 向用户报告： + - 问题根因分析 + - 修复建议 + - 是否建议再迭代一轮 + +### Phase 7: 记录到 Wiki + +1. 创建 finding 实体： + ```bash + conda run -n PolyGateway python3 .claude/tools/research_wiki.py add_entity research-wiki/ --type finding --id eval-<run_id> --title "Harness 评估: <run_id>" + ``` +2. 在 finding 中记录完整评估报告（硬性指标表 + 语义评估结果 + 诊断结论） +3. 建立 edge： + ```bash + conda run -n PolyGateway python3 .claude/tools/research_wiki.py add_edge research-wiki/ --from "finding:eval-<run_id>" --to "metric:<id>" --type evaluates --evidence "..." + ``` +4. 如果通过且用户确认，更新 metric 实体中的基线值为当前值 +5. 重建索引 + +## 输出格式 + +评估完成后，向用户输出结构化报告： + +| 字段 | 内容 | +|------|------| +| 运行信息 | run_id, git_sha, 耗时 | +| 硬性指标表 | 指标名, 基线, 当前, 阈值, 判定 | +| 语义评估表 | 维度, 得分, 判定, 说明 | +| 综合判定 | 通过/不通过 | +| 诊断（如有） | 根因, 建议 | + +## 与 subagent-driven-development 集成 + +在 SKILL.md 的 Step 9（final whole-implementation review）之后追加 Step 10： + +``` +Step 10: Harness 评估 + 调用 /harness-eval skill + 通过 → 进入 finishing-a-development-branch + 不通过 → 根据诊断结果，回到 Step 2 迭代相关 task +``` + +## 评估范围 + +本 skill 当前覆盖①工程评估；②提取信度/③诊断效度的全量评估待 `app/` 流水线落地后接入。 diff --git a/.claude/skills/idea-creator/SKILL.md b/.claude/skills/idea-creator/SKILL.md new file mode 100644 index 0000000..1cf735c --- /dev/null +++ b/.claude/skills/idea-creator/SKILL.md @@ -0,0 +1,108 @@ +--- +name: idea-creator +description: "Generate and rank research ideas. GPT-5.4 + Claude dual independent brainstorming. Trigger phrases: find ideas, brainstorm ideas, generate research ideas." +argument-hint: [research-direction] +--- + +# Research Idea Generator +Research direction: $ARGUMENTS + +## Constants +- `OUTPUT_DIR = idea-stage/` + +## Workflow + +### Phase 0: Load Wiki Context +If `research-wiki/query_pack.md` exists and was updated less than 7 days ago: +- Read it +- Treat failed ideas as a banlist +- Treat gaps as seeds +- Treat top papers as known prior work + +### Phase 1: Domain Survey +If there is no `query_pack`, do a quick WebSearch first to build a broad domain context. + +### Phase 2: Dual Independent Brainstorming +**Critical requirement: brainstorm independently on two tracks first, then synthesize.** + +GPT brainstorming (via Codex): +- Use `/codex:rescue --fresh --background` +- The prompt must ask GPT to generate 8-12 ideas +- The prompt must include: the domain overview, known gaps, and resource constraints +- Poll with `/codex:status` +- Use `/codex:result` to collect the output + +Claude brainstorming (via Agent): +- Dispatch a subagent through the Agent tool with `subagent_type="general-purpose"` +- Provide the same domain overview context +- Require it to independently generate 8-12 ideas + +### Phase 3: Claude Synthesis +- Merge both idea sets and deduplicate by similarity +- Cross-check: Claude challenges GPT ideas, GPT challenges Claude ideas +- Score and rank everything in a single pass +- Label each idea with its source: `GPT` / `Claude` / `consensus` + +### Phase 4: First Filtering Pass +Filtering criteria: +- Feasibility: compute, data, and implementation cost +- Fast novelty check: run 2-3 WebSearch passes for each idea +- Impact: answer "so what?" +- Remove ideas that are infeasible or too vague + +### Phase 5: Deep Verification +For the top ideas, call the `/novelty-check` skill. + +### Phase 7: Output Report +Write `idea-stage/IDEA_REPORT.md` in Chinese, with the following format: + +```md +# Research Idea Report +**Direction**: ... +**Date**: YYYY-MM-DD +**Evaluation Result**: X generated -> Y survived -> W recommended + +## Domain Overview Summary +## Recommended Ideas (Ranked) +### Idea 1: <title> +- Hypothesis / minimal experiment / novelty / feasibility / risk / source + +## Rejected Ideas +``` + +### Phase 8: Wiki Integration +- Check whether `research-wiki/` exists +- For each idea: + - `.claude/tools/research_wiki.py add_entity research-wiki/ --type idea --id <id> --title "..."` +- Add edges: + - `inspired_by` + - `addresses_gap` +- Rebuild `query_pack` and the index + +## Key Rules +- If the direction is too broad, STOP and ask the user to narrow it +- Failed ideas must also be written into the wiki +- Empirical signals matter more than theoretical attractiveness +- `"Apply X to Y"` is the lowest-level research idea +- If `research-wiki/` does not exist, still finish the report output but skip all wiki writes +- First independent, then synthesize; first evidence, then judgment; first filter, then dig deeper + +## Output Requirements +When the task is complete, you must provide at least: +- A domain overview summary +- 8-12 raw ideas +- A deduplicated candidate list +- Filtering results and reasons +- Deep verification results for the top ideas +- `idea-stage/IDEA_REPORT.md` +- Wiki write results, if applicable + +## Completion Criteria +Only finish when all of the following are complete: +1. Wiki context loading or domain survey is complete +2. Both independent brainstorming tracks are complete +3. Merge, deduplication, and ranking are complete +4. The first filtering pass is complete +5. Deep verification of top ideas is complete +6. `idea-stage/IDEA_REPORT.md` has been written +7. `research-wiki/` integration is complete, if applicable diff --git a/.claude/skills/novelty-check/SKILL.md b/.claude/skills/novelty-check/SKILL.md new file mode 100644 index 0000000..4c7fbeb --- /dev/null +++ b/.claude/skills/novelty-check/SKILL.md @@ -0,0 +1,141 @@ +--- +name: novelty-check +description: "Verify the novelty of research ideas. GPT cross-validation. Trigger phrases: novelty check, has anyone done this, check novelty." +argument-hint: [method-or-idea-description] +--- + +# Novelty Verification +Verify novelty of: $ARGUMENTS + +## Goal +Perform a strict check on whether a method, idea, or experimental setting is actually new. The default stance is skepticism, not help-seeking for supporting evidence. + +## Working Principles +- Brutally honest: do not relax the standard just to make something look new. +- `Applying X to Y` is not novel by default unless the application produces an unexpected mechanism, theoretical explanation, or clearly different experimental phenomenon. +- Check the novelty of both the `METHOD` and the `EXPERIMENTAL SETTING`. +- If the method itself is not new, but the findings, conclusions, experimental setup, or failure analysis are new, state that distinction explicitly. +- Always search the last 6 months of arXiv. +- Do not rely on titles alone; read the abstract and, when necessary, the key parts of related work, intro, method, and appendix. + +## Workflow + +### Phase A: Extract Core Claims +First break the user's method description into 3-5 core technical claims. Each one should be as specific as possible. + +For each claim, answer: +- What is the method? +- What problem does it solve? +- What is the key mechanism? +- What is the essential difference from an obvious baseline? + +Rewrite the story-like description into searchable technical propositions and avoid vague phrasing. + +### Phase B: Multi-source Literature Search +Run multi-source retrieval for each claim, prioritizing recent work and similar settings. + +For each claim, try at least 3 search-query sets, and make them complementary: +- Direct technical terms +- Synonyms / abbreviations / related task names +- "Problem + mechanism" combinations +- "Method + dataset / setting" combinations + +#### Required Search Channels +1. WebSearch: arXiv / Google Scholar / Semantic Scholar / conference homepages +2. `python3 .claude/tools/arxiv_fetch.py search "QUERY" --max 10` +3. `python3 .claude/tools/semantic_scholar_fetch.py search "QUERY" --max 10` +4. `python3 .claude/tools/exa_search.py search "QUERY" --max 10` (if available) +5. `python3 .claude/tools/openalex_fetch.py search "QUERY" --max 10` (if available) + +#### Search Priorities +- ICLR / NeurIPS / ICML 2025-2026 +- arXiv preprints from the last 6 months +- Papers close to the method mechanism, not only papers on the same task +- Papers close to the experimental setting, not only papers using the same method + +#### Decision Strategy +- Record potentially overlapping papers first; do not exclude them too early +- Prefer reading the abstract, intro, related work, and method sections +- If overlap looks suspicious, also read the experimental setup and appendix + +#### Recording Requirements +For each candidate paper, record: +- Title +- Year +- Venue / status +- Relevant point +- The specific reason it may overlap +- Why it might still be a different work + +If a data source is unavailable, explicitly record the fallback reason and continue with the others; do not stop the task. + +### Phase C: GPT Cross-Validation +Send the method description from Phase A and all candidate papers found in Phase B to `/codex:rescue --fresh --wait` for a second review. + +The cross-validation prompt must include: +- proposed method description +- the full candidate paper list +- ask: + - `Is this method novel?` + - `What is the closest prior work?` + - `What is the delta?` + +Use high reasoning effort. + +The goal of cross-validation is not to find even more papers. It is to force out the closest prior art, the smallest difference, and the risk of pseudo-novelty. + +### Phase D: Output Report + Wiki Integration +The output must be in English and follow a fixed structure. + +#### Report Format +```markdown +## Novelty Check Report +### Method Under Review +### Core Claims +- Claim 1: ... (novelty: high / medium / low; closest paper: ...) +- Claim 2: ... (novelty: high / medium / low; closest paper: ...) + +### Recent Prior Work +| Paper | Year | Venue / Status | Overlap Point | Key Difference | +|---|---:|---|---|---| + +### Overall Assessment +- score X/10 +- recommendation: continue / continue cautiously / abandon +- key differentiator: ... +- positioning advice: ... +``` + +#### Evaluation Scale +- `high`: current search shows no close prior art, and the difference is concrete and technical +- `medium`: there is related prior work, but there is still a clear and defensible technical delta +- `low`: mostly a reorganization of known methods, task switching, dataset switching, hyperparameter changes, or standard engineering changes + +#### Wiki Integration +If the project has `research-wiki/`, also ingest the knowledge there: +- Create a `claim` entity for each core claim +- Create a `paper` entity for each newly found paper +- Add claim-paper / paper-paper relation edges +- Rebuild `query_pack` + +Prefer existing tools such as `.claude/tools/research_wiki.py`; if the wiki does not exist, skip silently and do not error. + +Ingestion rules: +- Only write high-confidence information +- Claim names should be short, stable, and reusable +- Edges must include evidence; do not create empty links + +## Completion Criteria +Only finish when all of the following are complete: +1. 3-5 core claims have been extracted +2. Multi-source search has been completed, including the last 6 months of arXiv +3. Candidate paper abstracts have been read, and related work / method sections were read when necessary +4. GPT cross-validation has been completed +5. A structured English report has been produced +6. If `research-wiki/` exists, the corresponding writes and `query_pack` rebuild have been completed + +## Failure Handling +- Missing tools: record the missing item and degrade gracefully +- Too few search results: expand synonyms, abbreviations, higher-level terms, and experimental settings +- Too many search results: prefer the most recent, most similar, and most likely overlapping work +- Conflicting evidence: read the original abstract and method sections first; do not rely on intuition diff --git a/.claude/skills/receiving-code-review/SKILL.md b/.claude/skills/receiving-code-review/SKILL.md new file mode 100644 index 0000000..1acc6e0 --- /dev/null +++ b/.claude/skills/receiving-code-review/SKILL.md @@ -0,0 +1,228 @@ +--- +name: receiving-code-review +description: Use when receiving code review feedback, before implementing suggestions, especially if feedback seems unclear or technically questionable - requires technical rigor and verification, not performative agreement or blind implementation +--- + +# Code Review Reception + +## Overview + +Code review requires technical evaluation, not emotional performance. + +**Core principle:** Verify before implementing. Ask before assuming. Technical correctness over social comfort. + +## The Response Pattern + +``` +WHEN receiving code review feedback: + +1. READ: Complete feedback without reacting +2. UNDERSTAND: Restate requirement in own words (or ask) +3. VERIFY: Check against codebase reality +4. EVALUATE: Technically sound for THIS codebase? +5. RESPOND: Technical acknowledgment or reasoned pushback +6. IMPLEMENT: One item at a time, test each +``` + +## Forbidden Responses + +**NEVER:** +- "You're absolutely right!" (explicit CLAUDE.md violation) +- "Great point!" / "Excellent feedback!" (performative) +- "Let me implement that now" (before verification) + +**INSTEAD:** +- Restate the technical requirement +- Ask clarifying questions +- Push back with technical reasoning if wrong +- Just start working (actions > words) + +## Handling Unclear Feedback + +``` +IF any item is unclear: + STOP - do not implement anything yet + ASK for clarification on unclear items + +WHY: Items may be related. Partial understanding = wrong implementation. +``` + +**Example:** +``` +your human partner: "Fix 1-6" +You understand 1,2,3,6. Unclear on 4,5. + +❌ WRONG: Implement 1,2,3,6 now, ask about 4,5 later +✅ RIGHT: "I understand items 1,2,3,6. Need clarification on 4 and 5 before proceeding." +``` + +## Source-Specific Handling + +### From your human partner +- **Trusted** - implement after understanding +- **Still ask** if scope unclear +- **No performative agreement** +- **Skip to action** or technical acknowledgment + +### From External Reviewers +``` +BEFORE implementing: + 1. Check: Technically correct for THIS codebase? + 2. Check: Breaks existing functionality? + 3. Check: Reason for current implementation? + 4. Check: Works on all platforms/versions? + 5. Check: Does reviewer understand full context? + +IF suggestion seems wrong: + Push back with technical reasoning + +IF can't easily verify: + Say so: "I can't verify this without [X]. Should I [investigate/ask/proceed]?" + +IF conflicts with your human partner's prior decisions: + Stop and discuss with your human partner first +``` + +**your human partner's rule:** "External feedback - be skeptical, but check carefully" + +## YAGNI Check for "Professional" Features + +``` +IF reviewer suggests "implementing properly": + grep codebase for actual usage + + IF unused: "This endpoint isn't called. Remove it (YAGNI)?" + IF used: Then implement properly +``` + +**your human partner's rule:** "You and reviewer both report to me. If we don't need this feature, don't add it." + +## Implementation Order + +``` +FOR multi-item feedback: + 1. Clarify anything unclear FIRST + 2. Then implement in this order: + - Blocking issues (breaks, security) + - Simple fixes (typos, imports) + - Complex fixes (refactoring, logic) + 3. Test each fix individually + 4. Verify no regressions +``` + +## When To Push Back + +Push back when: +- Suggestion breaks existing functionality +- Reviewer lacks full context +- Violates YAGNI (unused feature) +- Technically incorrect for this stack +- Legacy/compatibility reasons exist +- Conflicts with your human partner's architectural decisions + +**How to push back:** +- Use technical reasoning, not defensiveness +- Ask specific questions +- Reference working tests/code +- Involve your human partner if architectural + + +## Acknowledging Correct Feedback + +When feedback IS correct: +``` +✅ "Fixed. [Brief description of what changed]" +✅ "Good catch - [specific issue]. Fixed in [location]." +✅ [Just fix it and show in the code] + +❌ "You're absolutely right!" +❌ "Great point!" +❌ "Thanks for catching that!" +❌ "Thanks for [anything]" +❌ ANY gratitude expression +``` + +**Why no thanks:** Actions speak. Just fix it. The code itself shows you heard the feedback. + +**If you catch yourself about to write "Thanks":** DELETE IT. State the fix instead. + +## Gracefully Correcting Your Pushback + +If you pushed back and were wrong: +``` +✅ "You were right - I checked [X] and it does [Y]. Implementing now." +✅ "Verified this and you're correct. My initial understanding was wrong because [reason]. Fixing." + +❌ Long apology +❌ Defending why you pushed back +❌ Over-explaining +``` + +State the correction factually and move on. + +## Common Mistakes + +| Mistake | Fix | +|---------|-----| +| Performative agreement | State requirement or just act | +| Blind implementation | Verify against codebase first | +| Batch without testing | One at a time, test each | +| Assuming reviewer is right | Check if breaks things | +| Avoiding pushback | Technical correctness > comfort | +| Partial implementation | Clarify all items first | +| Can't verify, proceed anyway | State limitation, ask for direction | + +## Real Examples + +**Performative Agreement (Bad):** +``` +Reviewer: "Remove legacy code" +❌ "You're absolutely right! Let me remove that..." +``` + +**Technical Verification (Good):** +``` +Reviewer: "Remove legacy code" +✅ "Checking... build target is 10.15+, this API needs 13+. Need legacy for backward compat. Current impl has wrong bundle ID - fix it or drop pre-13 support?" +``` + +**YAGNI (Good):** +``` +Reviewer: "Implement proper metrics tracking with database, date filters, CSV export" +✅ "Grepped codebase - nothing calls this endpoint. Remove it (YAGNI)? Or is there usage I'm missing?" +``` + +**Unclear Item (Good):** +``` +your human partner: "Fix items 1-6" +You understand 1,2,3,6. Unclear on 4,5. +✅ "Understand 1,2,3,6. Need clarification on 4 and 5 before implementing." +``` + +## GitHub Thread Replies + +When replying to inline review comments on GitHub, reply in the comment thread (`gh api repos/{owner}/{repo}/pulls/{pr}/comments/{id}/replies`), not as a top-level PR comment. + +## The Bottom Line + +**External feedback = suggestions to evaluate, not orders to follow.** + +Verify. Question. Then implement. + +No performative agreement. Technical rigor always. + +## Wiki Integration + +**Precondition**: `research-wiki/` directory exists (skip this section entirely if it does not). + +**Trigger**: When review feedback is received and you decide whether to accept or reject it. + +**Type**: `review` + +**Severity**: `approved` / `needs_changes` / `rejected` + +**Steps**: +1. Run `.claude/tools/research_wiki.py add_entity research-wiki/ --type review --id <slug> --title "<review response title>"` to create the review entity +2. Append the accepted or rejected suggestions, the reason, and the follow-up actions to the generated page +3. If the response changes a design or plan, run `.claude/tools/research_wiki.py add_edge research-wiki/ --from "review:<id>" --to "<target-type>:<id>" --type informs --evidence "..."` +4. Run `.claude/tools/research_wiki.py rebuild_index research-wiki/` diff --git a/.claude/skills/requesting-code-review/SKILL.md b/.claude/skills/requesting-code-review/SKILL.md new file mode 100644 index 0000000..a6a08b0 --- /dev/null +++ b/.claude/skills/requesting-code-review/SKILL.md @@ -0,0 +1,116 @@ +--- +name: requesting-code-review +description: Use when completing tasks, implementing major features, or before merging to verify work meets requirements +--- + +# Requesting Code Review + +Dispatch Codex as a read-only reviewer to catch issues before they cascade. The reviewer gets precisely crafted context for evaluation — never your session's history. This keeps the reviewer focused on the work product, not your thought process, and preserves your own context for continued work. + +**Core principle:** Review early, review often. + +## When to Request Review + +**Mandatory(硬门):** +- After completing a major feature +- Before merge to main (subagent-driven development 场景即其合并前一次性审查) + +**Optional but valuable:** +- When stuck (fresh perspective) +- Before refactoring (baseline check) +- After fixing complex bug + +## How to Request + +**1. Get git SHAs:** +```bash +BASE_SHA=$(git rev-parse HEAD~1) # or origin/main +HEAD_SHA=$(git rev-parse HEAD) +``` + +**2. Dispatch Codex as reviewer:** + +用 `/codex:rescue --fresh --wait` 把 `code-reviewer.md` 模板（含 `{BASE_SHA}`/`{HEAD_SHA}` 占位与 `git diff` 指令）交 Codex 只读审查。 + +**Placeholders:** +- `{DESCRIPTION}` - Brief summary of what you built +- `{PLAN_OR_REQUIREMENTS}` - What it should do +- `{BASE_SHA}` - Starting commit +- `{HEAD_SHA}` - Ending commit + +**3. Act on feedback:** +- Fix Critical issues immediately +- Fix Important issues before proceeding +- Note Minor issues for later +- Push back if reviewer is wrong (with reasoning) + +## Example + +``` +[Just completed Task 2: Add verification function] + +You: Let me request code review before proceeding. + +BASE_SHA=$(git log --oneline | grep "Task 1" | head -1 | awk '{print $1}') +HEAD_SHA=$(git rev-parse HEAD) + +[Dispatch Codex as reviewer via /codex:rescue --fresh --wait] + DESCRIPTION: Added verifyIndex() and repairIndex() with 4 issue types + PLAN_OR_REQUIREMENTS: Task 2 from research-wiki/plans/deployment-plan.md + BASE_SHA: a7981ec + HEAD_SHA: 3df7661 + +[Codex returns]: + Strengths: Clean architecture, real tests + Issues: + Important: Missing progress indicators + Minor: Magic number (100) for reporting interval + Assessment: Ready to proceed + +You: [Fix progress indicators] +[Continue to Task 3] +``` + +## Integration with Workflows + +**Subagent-Driven Development:** +- 合并前一次整分支审查(见该 skill 的 merge-reviewer-prompt.md);任务级把关交给自动质量门 + +**Executing Plans:** +- Review after each task or at natural checkpoints +- Get feedback, apply, continue + +**Ad-Hoc Development:** +- Review before merge +- Review when stuck + +## Red Flags + +**Never:** +- 合并前跳过审查(小改动可自判不做中途审查,但合并/PR 前的审查是硬门) +- Ignore Critical issues +- Proceed with unfixed Important issues +- Argue with valid technical feedback + +**If reviewer wrong:** +- Push back with technical reasoning +- Show code/tests that prove it works +- Request clarification + +See template at: requesting-code-review/code-reviewer.md + +## Wiki Integration + +**Precondition**: `research-wiki/` directory exists (skip this section entirely if it does not). + +**Trigger**: When requesting a code review or when a clear review result has been received. + +**Type**: `review` + +**Severity**: `approved` / `needs_changes` / `rejected` + +**Steps**: +1. Run `.claude/tools/research_wiki.py add_entity research-wiki/ --type review --id <slug> --title "<review title>"` to create the review entity +2. Append the review conclusion, key issues, recommendations, and whether it passed to the generated page +3. If the review recommendations change a design or plan, run `.claude/tools/research_wiki.py add_edge research-wiki/ --from "review:<id>" --to "<target-type>:<id>" --type informs --evidence "..."` +4. Run `.claude/tools/research_wiki.py rebuild_index research-wiki/` diff --git a/.claude/skills/requesting-code-review/code-reviewer.md b/.claude/skills/requesting-code-review/code-reviewer.md new file mode 100644 index 0000000..36f07a9 --- /dev/null +++ b/.claude/skills/requesting-code-review/code-reviewer.md @@ -0,0 +1,175 @@ +# Code Reviewer Prompt Template + +Use this template when dispatching Codex as reviewer. + +**Purpose:** Review completed work against requirements and code quality standards before it cascades into more work. + +``` +/codex:rescue --fresh --wait (read-only review): + description: "Review code changes" + prompt: | + You are a Senior Code Reviewer with expertise in software architecture, + design patterns, and best practices. Your job is to review completed work + against its plan or requirements and identify issues before they cascade. + + ## What Was Implemented + + {DESCRIPTION} + + ## Requirements / Plan + + {PLAN_OR_REQUIREMENTS} + + ## Git Range to Review + + **Base:** {BASE_SHA} + **Head:** {HEAD_SHA} + + ```bash + git diff --stat {BASE_SHA}..{HEAD_SHA} + git diff {BASE_SHA}..{HEAD_SHA} + ``` + + ## What to Check + + **Plan alignment:** + - Does the implementation match the plan / requirements? + - Are deviations justified improvements, or problematic departures? + - Is all planned functionality present? + + **Code quality:** + - Clean separation of concerns? + - Proper error handling? + - Type safety where applicable? + - DRY without premature abstraction? + - Edge cases handled? + + **Architecture:** + - Sound design decisions? + - Reasonable scalability and performance? + - Security concerns? + - Integrates cleanly with surrounding code? + + **Testing:** + - Tests verify real behavior, not mocks? + - Edge cases covered? + - Integration tests where they matter? + - All tests passing? + + **Production readiness:** + - Migration strategy if schema changed? + - Backward compatibility considered? + - Documentation complete? + - No obvious bugs? + + **Runtime data & logging:** + - Are SqliteRunStore calls present at all critical code paths? + - Do log tables capture enough data for post-run diagnosis? + - Query Wiki schemas/ to understand which tables this code affects + - Query SQLite to check if metrics regressed after these changes + - Are log_event calls placed at error boundaries and state transitions? + + ## Calibration + + Categorize issues by actual severity. Not everything is Critical. + Acknowledge what was done well before listing issues — accurate praise + helps the implementer trust the rest of the feedback. + + If you find significant deviations from the plan, flag them specifically + so the implementer can confirm whether the deviation was intentional. + If you find issues with the plan itself rather than the implementation, + say so. + + ## Output Format + + ### Strengths + [What's well done? Be specific.] + + ### Issues + + #### Critical (Must Fix) + [Bugs, security issues, data loss risks, broken functionality] + + #### Important (Should Fix) + [Architecture problems, missing features, poor error handling, test gaps] + + #### Minor (Nice to Have) + [Code style, optimization opportunities, documentation polish] + + For each issue: + - File:line reference + - What's wrong + - Why it matters + - How to fix (if not obvious) + + ### Recommendations + [Improvements for code quality, architecture, or process] + + ### Assessment + + **Ready to merge?** [Yes | No | With fixes] + + **Reasoning:** [1-2 sentence technical assessment] + + ## Critical Rules + + **DO:** + - Categorize by actual severity + - Be specific (file:line, not vague) + - Explain WHY each issue matters + - Acknowledge strengths + - Give a clear verdict + + **DON'T:** + - Say "looks good" without checking + - Mark nitpicks as Critical + - Give feedback on code you didn't actually read + - Be vague ("improve error handling") + - Avoid giving a clear verdict +``` + +**Placeholders:** +- `{DESCRIPTION}` — brief summary of what was built +- `{PLAN_OR_REQUIREMENTS}` — what it should do (plan file path, task text, or requirements) +- `{BASE_SHA}` — starting commit +- `{HEAD_SHA}` — ending commit + +**Reviewer returns:** Strengths, Issues (Critical / Important / Minor), Recommendations, Assessment + +## Example Output + +``` +### Strengths +- Clean database schema with proper migrations (db.ts:15-42) +- Comprehensive test coverage (18 tests, all edge cases) +- Good error handling with fallbacks (summarizer.ts:85-92) + +### Issues + +#### Important +1. **Missing help text in CLI wrapper** + - File: index-conversations:1-31 + - Issue: No --help flag, users won't discover --concurrency + - Fix: Add --help case with usage examples + +2. **Date validation missing** + - File: search.ts:25-27 + - Issue: Invalid dates silently return no results + - Fix: Validate ISO format, throw error with example + +#### Minor +1. **Progress indicators** + - File: indexer.ts:130 + - Issue: No "X of Y" counter for long operations + - Impact: Users don't know how long to wait + +### Recommendations +- Add progress reporting for user experience +- Consider config file for excluded projects (portability) + +### Assessment + +**Ready to merge: With fixes** + +**Reasoning:** Core implementation is solid with good architecture and tests. Important issues (help text, date validation) are easily fixed and don't affect core functionality. +``` diff --git a/.claude/skills/research-lit/SKILL.md b/.claude/skills/research-lit/SKILL.md new file mode 100644 index 0000000..851e649 --- /dev/null +++ b/.claude/skills/research-lit/SKILL.md @@ -0,0 +1,213 @@ +--- +name: research-lit +description: "Search and analyze research papers, literature reviews, and related work. Trigger phrases: find papers, literature review, related work, literature review." +argument-hint: [research-topic] +--- + +# Literature Review +Research topic: $ARGUMENTS + +## Constants +- `PAPER_LIBRARY = references/` (local PDF directory) +- `MAX_LOCAL_PAPERS = 20` + +## Data Sources (all enabled by default) + +| Data source | How to determine availability | What it provides | Fallback behavior | +|---|---|---|---| +| Local PDF | `references/` exists and `references/**/*.pdf` is non-empty | Papers, reports, appendices, and drafts collected in the repo | Read only locally; if there are no PDFs, continue with online search | +| Web search | Network is available and general search results are accessible | Google Scholar / paper pages / arXiv / conference homepages / survey blogs | If search fails, skip that source and keep results from others | +| arXiv API | `python3 .claude/tools/arxiv_fetch.py` is runnable and the network is available | arXiv metadata, abstracts, IDs, categories, versions | Skip if unavailable; prefer arXiv records from other sources | +| Semantic Scholar | `python3 .claude/tools/semantic_scholar_fetch.py` is runnable and the network is available | Paper metadata, venue, citation, author, and citation relationships | Skip if unavailable; if an arXiv paper has S2 venue metadata, prefer S2 | +| Exa | `python3 .claude/tools/exa_search.py` is runnable, the network is available, and API config exists | Semantic search, page highlight excerpts, related paper page clues | Skip if unavailable; use results only as supplemental clues | +| OpenAlex | `python3 .claude/tools/openalex_fetch.py` is runnable and the network is available | Open scholarly graph, DOI, venue, year, and citation relationships | Skip if unavailable; use it to fill in DOI / venue / author information | +| DeepXiv | `python3 .claude/tools/deepxiv_fetch.py` is runnable and the network is available | Semantic search and paper aggregation results focused on arXiv | Skip if unavailable; cross-check against arXiv / S2 | + +### Coverage Control +All data sources are enabled by default. If the user includes the instruction: + +`— sources: <list>` + +then only the sources listed in `<list>` are used. Parsing rules: + +- `<list>` is comma-separated and may use Chinese or English names, such as `local PDF, arXiv, Semantic Scholar` +- Keep only recognizable names; ignore unknown items +- If parsing yields nothing, fall back to the default of enabling everything +- Either local-only reading or online-only searching is allowed; do not stop because some sources are unavailable + +## Workflow + +### Step 0: Scan local PDFs + +Scan the local library first, then decide where to focus online search. + +1. Glob: `references/**/*.pdf` +2. Filter by relevance to the research topic, prioritizing PDFs whose title, abstract, first chapter, or conclusion matches the topic +3. For up to `MAX_LOCAL_PAPERS` relevant PDFs, read the first 3 pages +4. Record the title, authors, year, venue, method keywords, and relevance for each paper + +Recommended reading command: + +```bash +python3 tools/read_pdf_pages.py references/path/to/paper.pdf --pages 1-3 +``` + +If the repository does not include that script, use any available PDF-reading tool or existing command. The rule is to read only the first 3 pages and avoid blind full-document reading. + +### Step 1: Online search + +Run retrieval for each enabled data source. Commands should be structured as closely as possible to the following forms. `QUERY` should be replaced with a search string built around the research topic, including the task noun, core method, aliases, synonyms, and common abbreviations. + +- `python3 .claude/tools/arxiv_fetch.py search "QUERY" --max 10` +- `python3 .claude/tools/semantic_scholar_fetch.py search "QUERY" --max 10` +- `python3 .claude/tools/exa_search.py search "QUERY" --max 10 --category "research paper" --content highlights` +- `python3 .claude/tools/openalex_fetch.py search "QUERY" --max 10 --year "2022-"` +- `python3 .claude/tools/deepxiv_fetch.py search "QUERY" --max 10` +- WebSearch for Google Scholar / the general web + +Search strategy: + +1. Start with broad queries to identify the main direction +2. Use narrower queries to find recent work, SOTA, benchmarks, surveys, and ablations from the last two years +3. Add controversy points, failure cases, negative results, and competing methods +4. Record raw results from each source; do not drop borderline candidates too early + +### Step 2: Cross-source deduplication + +Deduplicate in this order: + +1. `arXiv ID` +2. `DOI` +3. normalized title + +Rules: + +- Normalize titles by lowercasing, removing punctuation, removing extra spaces, and removing version suffixes +- If the same paper has different metadata across sources, keep the record with the most complete information +- If `S2` (Semantic Scholar) provides venue, year, author, or citation metadata for an arXiv paper, prefer those fields from `S2` +- Distinguish between preprints and formally published versions; if both correspond to the same research, record the relationship but ultimately prefer the more authoritative published version + +### Step 3: Analyze each paper + +For each retained paper, extract: + +- `problem/gap`: what problem is being solved and what existing methods are missing +- `method`: core idea, model, training / inference flow, and key tricks +- `key results`: main experimental findings, metrics, baselines, and datasets +- `relevance to our work`: the direct connection to the current research topic and what can be borrowed +- `source`: which source or sources the paper came from and whether metadata conflicts exist + +Requirements: + +- Every paper must include author, year, and venue +- Explicitly mark `preprint`, `conference paper`, `journal paper`, `workshop`, and similar statuses +- If author / year / venue is uncertain, state the source of uncertainty and do not fabricate it + +### Step 4: Synthesis + +Cluster papers by method route or theme instead of sorting only by time. + +The synthesis must answer: + +- Which method routes have become the consensus +- Where the important disagreements are +- Which conclusions hold only for specific datasets or settings +- What gaps remain unsolved +- Which results matter most for our research, and why + +Prioritize: + +- Work from the last 2 years +- Representative methods and accepted baselines for the direction +- Ablations, diagnostics, and failure analyses that explain performance differences + +If the topic is foundational or classical, trace back to the original work, but keep the newest work as the main line. + +### Step 5: Output + +The final output must include both: + +1. A structured literature table +2. A 3-5 paragraph narrative summary + +Suggested table columns: + +- Paper +- Authors / Year / Venue +- Source +- Problem / Gap +- Method +- Key Results +- Relevance to Our Work + +The narrative should: + +- Summarize the sub-branches of the topic first, then the consensus and disagreements +- Clearly call out the 3-5 papers worth following next +- Clearly identify 1-3 gaps that can become future research entry points + +### Step 6: Wiki Integration + +First check whether `research-wiki/` exists; if it does not, skip all writes without error. + +If it does exist, then: + +1. Ingest the top 8-12 papers into the wiki + +```bash +.claude/tools/research_wiki.py ingest_paper research-wiki/ --arxiv-id <id> [--title "..." --authors "..." --year ...] +``` + +2. Create entities for identified gaps + +```bash +.claude/tools/research_wiki.py add_entity research-wiki/ --type gap --id <slug> --title "..." +``` + +3. Add paper relationship edges + +```bash +.claude/tools/research_wiki.py add_edge research-wiki/ --from "paper:X" --to "paper:Y" --type extends --evidence "..." +``` + +4. Rebuild the query pack and index + +```bash +.claude/tools/research_wiki.py rebuild_query_pack research-wiki/ && .claude/tools/research_wiki.py rebuild_index research-wiki/ +``` + +Integration rules: + +- Only write papers with high confidence and highest topic relevance into the wiki +- Gap node names should be short and stable so they can be reused later +- Relationship edges must include evidence; do not create vague connections + +## Key Rules + +- Always cite papers: author, year, and venue are required +- Distinguish peer-reviewed papers from preprints +- Missing tools, missing APIs, or missing network access must degrade gracefully; never stop because one source fails +- Focus on the last 2 years by default; only trace back further when the task is about foundational work +- Do not just list papers; summarize the thread, disagreements, and gaps +- If the query is too narrow, first expand with synonyms, abbreviations, and higher-level terms, then narrow again to a precise sub-direction +- If there are too many results, keep representative, highly cited, newer, stronger-experiment, and closest-to-topic papers first +- If there are too few results, broaden the query and rescan relevant PDFs in the local library + +## Execution Standard + +- Local first, then online +- Deduplicate first, then analyze +- Evidence first, then conclusions +- Cluster first, then narrate +- Integrate the wiki first, then finish + +## Completion Criteria + +Only finish the task when all of the following are complete: + +1. Local PDF scanning is complete +2. At least the default-enabled data sources were covered, or fallback reasons were recorded +3. Deduplication is complete +4. Every core paper has been analyzed +5. A table and narrative summary were produced +6. If `research-wiki/` exists, the corresponding writes and rebuilds were completed diff --git a/.claude/skills/research-wiki/SKILL.md b/.claude/skills/research-wiki/SKILL.md new file mode 100644 index 0000000..13eae94 --- /dev/null +++ b/.claude/skills/research-wiki/SKILL.md @@ -0,0 +1,89 @@ +--- +name: research-wiki +description: "Manage the project research knowledge base. Initialization, querying, statistics, and health checks. Trigger phrases: knowledge base, research wiki, wiki query, check the knowledge base." +argument-hint: [subcommand: init|query|stats|lint|ingest] +--- + +# Research Wiki Management + +Manage the project research knowledge base: $ARGUMENTS + +## Overview + +Research Wiki is the project's single source of truth for knowledge. All research and development knowledge produced by skills is stored here. + +### Entity Types (12) + +| Entity | Directory | node_id Format | Primary Source Skill | +|---|---|---|---| +| paper | `papers/` | `paper:<slug>` | `research-lit` | +| idea | `ideas/` | `idea:<id>` | `idea-creator` | +| experiment | `experiments/` | `exp:<id>` | `run-experiment` (future) | +| claim | `claims/` | `claim:<id>` | `novelty-check` | +| gap | `gaps/` | `gap:<id>` | `research-lit` / `idea-creator` | +| design | `designs/` | `design:<id>` | `brainstorming` | +| finding | `findings/` | `finding:<id>` | `systematic-debugging` / `verification-before-completion` | +| adr | `adrs/` | `adr:<id>` | 架构决策记录 | +| plan | `plans/` | `plan:<id>` | `writing-plans` | +| review | `reviews/` | `review:<id>` | `requesting-code-review` / `receiving-code-review` | +| schema | `schemas/` | `schema:<id>` | `structured-logging` | +| metric | `metrics/` | `metric:<id>` | `structured-logging` / `harness-eval` | + +### Key Files + +| File | Purpose | +|---|---| +| `query_pack.md` | Compressed summary | +| `index.md` | Category index | +| `log.md` | Audit log | +| `graph/edges.json` | Relationship graph | + +## Subcommands + +### `/research-wiki init` + +Initialize the wiki: + +```bash +.claude/tools/research_wiki.py init research-wiki/ +``` + +### `/research-wiki query "<topic>"` + +Rebuild `query_pack.md`: + +```bash +.claude/tools/research_wiki.py rebuild_query_pack research-wiki/ +``` + +Then show the content to the user. + +### `/research-wiki stats` + +Show statistics: + +```bash +.claude/tools/research_wiki.py stats research-wiki/ +``` + +### `/research-wiki lint` + +Run health checks: + +```bash +.claude/tools/research_wiki.py lint research-wiki/ +``` + +### `/research-wiki ingest "<title>" — arxiv: <id>` + +Import a paper: + +```bash +.claude/tools/research_wiki.py ingest_paper research-wiki/ --arxiv-id <id> +``` + +## Key Rules + +- Other skills call `.claude/tools/research_wiki.py` directly; they do not go through this skill. +- If the wiki does not exist (`research-wiki/` directory is missing), all write operations are skipped silently. +- Failed ideas are always preserved in `query_pack`. diff --git a/.claude/skills/structured-logging/SKILL.md b/.claude/skills/structured-logging/SKILL.md new file mode 100644 index 0000000..a382569 --- /dev/null +++ b/.claude/skills/structured-logging/SKILL.md @@ -0,0 +1,50 @@ +--- +name: structured-logging +description: "设计结构化日志/遥测方案。当功能会产生运行时数据时,在 brainstorming 产出 design 之后、writing-plans 之前调用:确定记录什么、记到哪、如何评估,并注册到 Wiki。纯内部重构、不产生运行时数据的改动不需要。" +argument-hint: "[功能描述]" +--- + +# Structured Logging + +## Overview + +为即将开发的功能设计结构化日志/遥测方案:记录什么数据、落到哪张表、如何评估。 + +**边界**: 会产生运行时数据的功能,编码前必须有日志方案——埋点是"当前需要"(CLAUDE.md P1),事后补埋点意味着丢失基线数据。不产生运行时数据的改动直接跳过本 skill。 + +本项目背景:PolyGateway 自带遥测子系统(`telemetry/`,SQLite 后端,每次调用必录,见 CLAUDE.md 库铁律"遥测必录")。本 skill 设计的是**具体功能的埋点方案**,必须与遥测子系统对齐,不另起炉灶。 + +## 设计要回答的问题 + +1. **这个功能产生什么运行时数据?** 网关领域的典型维度: + - 每次调用的时延 / TTFT / token 用量与成本 + - 错误分类计数(Transient/SourceDead/RequestRejected/ResultInvalid)与重试次数 + - 熔断状态迁移、限流等待/拒绝、缓存命中率 + - 阶段事件(开始/结束/错误/降级) +2. **新建表还是复用现有表?** 现有表能覆盖就复用;需要新维度才新建。先对照 `research-wiki/schemas/` 已登记的 schema 与实际库中的表,标记不一致。 +3. **每张表的 schema**: 列名、类型、说明、主键、哪些列服务于诊断查询。 +4. **埋点位置**: 具体模块与函数;必须走库的 `TelemetryRecorder` 端口/统一 helper,禁止散落的 ad-hoc 写库(三项目遥测调用被复制 4 次的教训)。 +5. **评估基线**: 可量化指标 + 阈值 + 判定方式;基线来源(对比哪次历史运行;首次则标"待首次运行后建立")。 + +## 注册到 Wiki(留痕,不可省略) + +```bash +.claude/tools/research_wiki.py add_entity research-wiki/ --type schema --id <table-name> --title "表结构: <table-name>" +.claude/tools/research_wiki.py add_entity research-wiki/ --type metric --id <metric-name> --title "<指标描述>" +.claude/tools/research_wiki.py add_edge research-wiki/ --from "metric:<id>" --to "schema:<id>" --type measures --evidence "..." +.claude/tools/research_wiki.py add_edge research-wiki/ --from "schema:<id>" --to "design:<id>" --type implements --evidence "..." +.claude/tools/research_wiki.py rebuild_index research-wiki/ +``` + +在生成的 md 中填入完整列定义、埋点位置、基线值与阈值。 + +## 产出(交给 writing-plans) + +埋点清单:哪些文件、哪些函数、在什么位置记什么。这些埋点必须成为 plan 中的显式步骤,不得遗漏。 + +| 产出 | 位置 | +|------|------| +| schema 实体 | `research-wiki/schemas/<name>.md` | +| metric 实体 | `research-wiki/metrics/<name>.md` | +| edge 关系 | `research-wiki/graph/edges.json` | +| 埋点清单 | 传递给 writing-plans | diff --git a/.claude/skills/subagent-driven-development/SKILL.md b/.claude/skills/subagent-driven-development/SKILL.md new file mode 100644 index 0000000..16b9015 --- /dev/null +++ b/.claude/skills/subagent-driven-development/SKILL.md @@ -0,0 +1,68 @@ +--- +name: subagent-driven-development +description: "Optional executor for large approved plans (many mostly-independent tasks): delegate each task to a fresh Claude subagent, run an automated quality gate per task, and one independent Codex review before merge. For small or tightly-coupled plans, implement directly instead." +--- + +# Subagent-Driven Development + +## 何时使用 + +- **适用**: 已有批准的 plan、任务多(≥3)且大体相互独立、值得为每个任务开独立上下文。 +- **不适用**: 小计划、任务强耦合、探索性工作——直接实现更省更好。 + +结构:Claude 主会话做控制器;每个任务派一个**全新** Claude subagent 实现;任务完成跑**自动质量门**;全部任务完成后、合并前做**一次** Codex 独立审查(跨模型,消除自评盲区)。 + +## 流程 + +### 1. 读计划,建任务清单 + +读一遍 plan,把每个任务的**全文**与上下文提取出来,进 TodoWrite。之后不再让 subagent 去读 plan 文件——派发时把任务全文直接贴进 prompt。 + +若项目已建 graphify 知识图谱(`graphify-out/` 存在),先 `/graphify . --update` 刷新;未建图则跳过,不阻断。 + +### 2. 派发实现 subagent(每任务一个,全新上下文) + +用 `Agent` 工具(`subagent_type=general-purpose`),prompt 用 `./claude-implementer-prompt.md` 模板填充(任务全文、上下文、绝对路径)。**记下 agentId**。 + +- 同一任务的返修一律 `SendMessage(to: agentId)` 发回原 subagent(保留其上下文);只有下一个任务才开新 subagent。 +- subagent 以结构化 `STATUS` 块收尾:`DONE` → 进质量门;`NEEDS_CONTEXT` → 补上下文继续;`BLOCKED` → 评估(补上下文/拆任务/计划有误则上报人类);缺失 STATUS 块按 `BLOCKED` 处理。 + +### 3. 自动质量门(每任务,机器判定) + +```bash +conda run -n PolyGateway ruff format --check <changed_files> +conda run -n PolyGateway ruff check <changed_files> +conda run -n PolyGateway radon cc <changed_files> -n C -s # C 级及以下复杂度即失败 +conda run -n PolyGateway pytest tests/ -x -q +# 结构检查: 无裸 except / except Exception: pass;核心文件不超 200 行 +``` + +失败 → 汇总工具输出,`SendMessage` 发回原 subagent 修,最多 2 轮,仍失败则上报人类。仅格式问题可由控制器直接 `ruff format` 修掉。测试超 30s 的在 tmux 里跑(`tmux new-session -d -s sdd-task<N> "<cmd>"`),便于人类 attach。 + +**不信任 subagent 自述**: 标记任务完成前,控制器亲自看 `git diff` 确认变更真实存在。 + +### 4. 合并前一次 Codex 独立审查(整分支) + +全部任务完成、质量门全绿后,用 `/codex:rescue --fresh --wait` 按 `./merge-reviewer-prompt.md` 做**一次**只读审查,范围是整条分支(所有 SHA + plan 全文),一次覆盖:spec 符合性(缺失/多余/误解)、跨任务集成问题、功能质量、明显的过度设计。 + +- Critical/Important 问题 → 发回对应 subagent(或自己)修复,复审至清零。 +- 前置:Codex 插件可用(`/codex:setup` 报 ready);审查模型用 `.codex/config.toml` 的默认强配置。Codex 不可用时,降级为派一个全新上下文的 Claude verifier subagent 按同一 prompt 审(见 `verification-before-completion`)。 + +### 5. 收尾 + +`/graphify . --update`(若在用),然后交给 `finishing-a-development-branch`。 + +## 纪律(约束点) + +- 连续执行,任务间不停下来找人类确认;只有 BLOCKED 无解、真歧义、全部完成三种停法。 +- 每任务提交留痕;严禁把多个任务squash成一坨再审。 +- 审查不接受"差不多就行":Critical/Important 清零才合并。 + +## Companion files + +- `./claude-implementer-prompt.md` — 实现 subagent 的 prompt 模板。 +- `./merge-reviewer-prompt.md` — 合并前一次性 Codex 审查的 prompt 模板。 + +## Wiki 留痕(`research-wiki/` 存在时) + +产生可复用实现知识或有价值审查意见的任务,记 plan/review 实体并连边(工具 `.claude/tools/research_wiki.py`,类型 `implements`/`informs`),纯机械改动跳过。 diff --git a/.claude/skills/subagent-driven-development/claude-implementer-prompt.md b/.claude/skills/subagent-driven-development/claude-implementer-prompt.md new file mode 100644 index 0000000..2a7a9ce --- /dev/null +++ b/.claude/skills/subagent-driven-development/claude-implementer-prompt.md @@ -0,0 +1,59 @@ +# Claude Subagent Implementer Prompt Template + +派发实现 subagent(`Agent` 工具,`subagent_type=general-purpose`)时,以下面模板为 prompt 主体,填好方括号内容。新任务开新 subagent;同一任务的返修用 `SendMessage(to: agentId)`。 + +--- + +``` +You are implementing Task N: [task name] in the PolyGateway repository. + +A controller spawned you, will read your output, verify your diff itself, and run an independent +cross-model review before merge. Reviewers read actual code — optimize for being right, not for +sounding right. + +## Task Description + +[任务全文,从 plan 逐字粘贴。不要给文件路径让 subagent 自己去找。] + +## Context + +[背景:该任务在整体设计中的位置、依赖、既有文件与约定。给足上下文,省一次 NEEDS_CONTEXT 往返。] + +## Working Directory + +[worktree 绝对路径] + +## Project Conventions (non-negotiable, gates enforce these) + +- 先读根目录 `CLAUDE.md`(库铁律、代码规范、目录规则)与 `research-wiki/ARCHITECTURE.md` 相关章节。 +- **目录**: 库代码只进 `src/polygateway/`(内核 ports/types/errors + middleware/transports/backends/telemetry/structured);测试进 `tests/{unit,integration,e2e}/`;`scripts/` 只放 `.sh`;根目录不得出现 `.py`;禁止 `helpers/ common/ shared/ misc/ lib/` 目录名。 +- **`reference/` 只读**,绝不修改(有 hook 硬拦截)。它是迁移蓝本:涉及迁移的任务必须逐段比对参考实现,不得简化核心逻辑。 +- **conda 环境**: 一切 Python 命令经 `conda run -n PolyGateway <cmd>`,不裸调 pytest/ruff。 +- **中文 Docstring**(模块/类/公共函数)、loguru 而非 print、公共函数完整类型注解、严禁裸 except 与 except Exception: pass、敏感信息只走 .env。 +- **测试证据**: 每个行为变更须有先失败后通过的测试证据(结果门);用真实样本或其二次构造。 +- 超过 30 秒的命令在 tmux 中运行(`tmux new-session -d -s sdd-taskN-<name> "<cmd>"`)。 +- 不做任务外的重构/抽象;文件长得不健康就在 NOTES 里标记,让控制器决定。 + +## Escalation + +你无法与控制器交互式提问。真被卡住时不要猜:以 `STATUS: NEEDS_CONTEXT`(缺信息)或 +`STATUS: BLOCKED`(无法完成)结束,并具体说明卡点、已尝试什么、需要什么帮助。 +非阻塞的判断题(如 helper 放哪),选最站得住脚的做法实现,并在 NOTES 里写明供审查者挑战。 + +## Your Job + +1. 精确实现任务所述——不多不少。 +2. 写测试并留下红→绿证据;跑 `conda run -n PolyGateway pytest` 与 ruff/radon,全绿。 +3. 按语义分段提交,常规提交信息格式(见 commit skill 规则,无 AI 签名)。 +4. 交付前自查:spec 每条都实现了吗?有没有 spec 外的东西?报告里的每句话对得上 diff 吗? + +## Report Format (required — last thing in your output) + +--- +STATUS: <DONE | DONE_WITH_CONCERNS | NEEDS_CONTEXT | BLOCKED> +COMMIT_SHAS: <short SHAs, or "none"> +FILES_CHANGED: <paths, or "none"> +TESTS: <"all passing: X/Y" | "failing: <details>" | "not applicable"> +NOTES: <STATUS != DONE 时必填;顾虑、判断题、具体卡点写这里> +--- +``` diff --git a/.claude/skills/subagent-driven-development/merge-reviewer-prompt.md b/.claude/skills/subagent-driven-development/merge-reviewer-prompt.md new file mode 100644 index 0000000..70267d9 --- /dev/null +++ b/.claude/skills/subagent-driven-development/merge-reviewer-prompt.md @@ -0,0 +1,61 @@ +# Merge Reviewer Prompt Template(合并前一次性独立审查) + +全部任务完成、自动质量门全绿后,用 `/codex:rescue --fresh --wait` 以本模板做**一次**只读审查,范围是整条分支。Codex 不可用时,同一模板派给全新上下文的 Claude verifier subagent(只读)。 + +--- + +``` +Codex review (read-only) — pass this as the /codex:rescue --fresh --wait prompt body: + +description: "Pre-merge review for <branch>" +prompt: | + You are the independent pre-merge reviewer for work implemented by Claude subagents in the + PolyGateway repository. You are read-only: read diffs and report, do not edit code. + + ## Plan / Requirements + + [plan 全文或其需求部分,逐字粘贴] + + ## Scope + + Working directory: [绝对路径] + Commits to review: [分支上全部 SHA,或 base..head 区间] + + ## Do Not Trust Reports + + 实现方的自述可能不完整或过于乐观。一切以 `git show <sha>`、`git diff <base>..<head>` 与直接 + 读文件为准,逐条独立核验。 + + ## Review Dimensions (one pass, all of them) + + 1. **Spec 符合性**: plan 每条需求能否指到具体实现行?有无缺失、多余(未要求的功能/flag/依赖)、 + 误解(接口/位置/签名与 plan 不符)?测试是在测 spec 要求的行为,还是在测"碰巧写出的代码"? + 有没有被删除或弱化的既有测试? + 2. **跨任务集成**: 任务间接口漂移、跨文件重复逻辑、早期任务留下的死代码、日志/配置/错误路径 + 的跨任务一致性——这些是单任务审查抓不到的。 + 3. **功能质量**: 职责划分、错误处理(吞异常/静默回退?)、并发与取消安全(CancelledError 穿透?)、 + 降级方向是否符合 CLAUDE.md 库铁律、测试是否覆盖边界而非只有 happy path。 + 4. **迁移保真**(若涉及 reference/ 蓝本迁移): 对照参考实现,核心逻辑(分支条件、Lua 语义、 + 状态机、退避公式)是否被简化或改变语义而未在设计中声明。 + 5. **明显过度设计**: 无人使用的参数/扩展点、可以是函数的类、不必要的间接层(YAGNI)。 + + ## Calibration + + 只报会造成真实问题的项。措辞偏好与格式吹毛求疵不报(自动工具已管)。 + - Critical — 真实 bug、数据损坏、安全问题、破坏下游迁移承诺。必修。 + - Important — 维护痛点、脆弱代码、spec 缺口。应修。 + - Minor — 顺手可改,不阻塞。 + + ## Report Format + + Verdict: <APPROVED | CHANGES_REQUESTED> + Spec gaps: <bullets 或 "none"> + Integration: <bullets 或 "none"> + Issues: + Critical: <file:line + 问题,或 "none"> + Important: <file:line + 问题,或 "none"> + Minor: <file:line + 问题,或 "none"> + Assessment: <一段话:整体质量、能否合并、跨切面问题> + + APPROVED = Critical 与 Important 均为 none。否则控制器将把发现发回修复并复审。 +``` diff --git a/.claude/skills/systematic-debugging/SKILL.md b/.claude/skills/systematic-debugging/SKILL.md new file mode 100644 index 0000000..d17e11a --- /dev/null +++ b/.claude/skills/systematic-debugging/SKILL.md @@ -0,0 +1,56 @@ +--- +name: systematic-debugging +description: Use when encountering any bug, test failure, or unexpected behavior, before proposing fixes - find the root cause first; symptom patches are failure +--- + +# Systematic Debugging + +## 边界声明 + +``` +先定位根因,再动手修。没有根因假设与证据,不提出修复。 +``` + +理由:乱试修复浪费时间且制造新 bug;症状修补会掩盖真实问题,在库场景下同时击穿所有下游项目。系统化定位对简单 bug 同样更快——简单 bug 也有根因。 + +## 定位要素(按需取用,不是仪式) + +1. **完整读错误信息与堆栈**——它经常直接给出答案。记下行号、文件、错误码。 +2. **稳定复现**——不能复现就先收集数据,不要猜。 +3. **查最近变更**——`git diff`、新依赖、配置与环境差异。已建 graphify 图谱时,可用 `conda run -n PolyGateway graphify path/affected/explain` 追调用链与影响面,免于盲读。 +4. **多组件系统先取证再归因**——在组件边界加日志(进/出数据、配置传播),跑一次拿到"断在哪一层"的证据,再深入该层。深层调用栈用回溯法(见 `root-cause-tracing.md`):坏值从哪来,一路向上追到源头,在源头修。 +5. **查结构化运行日志**——若项目有遥测/结构化日志(如 SQLite 遥测表、`research-wiki/schemas/` 登记的表),查事件流与指标趋势,让假设有数据支撑而非纯读码猜测。 +6. **对照可工作的样例**——同库相似可用代码、`reference/` 参考实现。逐项列差异,不要跳过"这不可能有影响"的差异;参考实现要读完整,不要按印象改编。 + +## 假设与修复(纪律) + +- 一次一个假设,写清"我认为根因是 X,因为 Y";用**最小改动**验证,一次只动一个变量。 +- 修复前先有失败的复现测试(接 `test-driven-development` 结果门);修复只针对根因,禁止"顺手"重构与打包多个改动。 +- 修完验证:该测试通过、其余测试不回归、原症状确实消失。 + +**3 次修复失败 = 停下质疑架构。** 若每次修复都在别处暴露新问题、或修复需要"大动干戈"才能实施,这不是假设错了,是架构错了——停止继续修,与人类讨论架构后再动。 + +若彻查后确属环境/时序/外部问题:记录调查过程,实现恰当的处理(重试/超时/报错),加监控埋点。但 95% 的"查无根因"是调查没做完。 + +## 红线(出现即停,回到定位) + +- "先快速修一下,回头再查" / "改改 X 试试看" / "大概是 X,先修了再说" +- 一次提交多个猜测性修改;注释掉测试或跳过校验让报错消失 +- 已经失败 2 次还想"再试一个修复" + +## 附属技术 + +- `root-cause-tracing.md` — 沿调用栈回溯到源头 +- `defense-in-depth.md` — 找到根因后的多层校验 +- `condition-based-waiting.md` — 用条件轮询替代拍脑袋超时 + +## Wiki 留痕(`research-wiki/` 存在时) + +得出值得留存的结论时(尤其影响后续任务的),记 finding: + +```bash +.claude/tools/research_wiki.py add_entity research-wiki/ --type finding --id <slug> --title "<问题>" +.claude/tools/research_wiki.py rebuild_index research-wiki/ +``` + +页内记录:症状、根因、验证方法、修复、影响面;若暴露 design/plan 缺陷,加 `reveals` 边。 diff --git a/.claude/skills/systematic-debugging/condition-based-waiting-example.ts b/.claude/skills/systematic-debugging/condition-based-waiting-example.ts new file mode 100644 index 0000000..703a06b --- /dev/null +++ b/.claude/skills/systematic-debugging/condition-based-waiting-example.ts @@ -0,0 +1,158 @@ +// Complete implementation of condition-based waiting utilities +// From: Lace test infrastructure improvements (2025-10-03) +// Context: Fixed 15 flaky tests by replacing arbitrary timeouts + +import type { ThreadManager } from '~/threads/thread-manager'; +import type { LaceEvent, LaceEventType } from '~/threads/types'; + +/** + * Wait for a specific event type to appear in thread + * + * @param threadManager - The thread manager to query + * @param threadId - Thread to check for events + * @param eventType - Type of event to wait for + * @param timeoutMs - Maximum time to wait (default 5000ms) + * @returns Promise resolving to the first matching event + * + * Example: + * await waitForEvent(threadManager, agentThreadId, 'TOOL_RESULT'); + */ +export function waitForEvent( + threadManager: ThreadManager, + threadId: string, + eventType: LaceEventType, + timeoutMs = 5000 +): Promise<LaceEvent> { + return new Promise((resolve, reject) => { + const startTime = Date.now(); + + const check = () => { + const events = threadManager.getEvents(threadId); + const event = events.find((e) => e.type === eventType); + + if (event) { + resolve(event); + } else if (Date.now() - startTime > timeoutMs) { + reject(new Error(`Timeout waiting for ${eventType} event after ${timeoutMs}ms`)); + } else { + setTimeout(check, 10); // Poll every 10ms for efficiency + } + }; + + check(); + }); +} + +/** + * Wait for a specific number of events of a given type + * + * @param threadManager - The thread manager to query + * @param threadId - Thread to check for events + * @param eventType - Type of event to wait for + * @param count - Number of events to wait for + * @param timeoutMs - Maximum time to wait (default 5000ms) + * @returns Promise resolving to all matching events once count is reached + * + * Example: + * // Wait for 2 AGENT_MESSAGE events (initial response + continuation) + * await waitForEventCount(threadManager, agentThreadId, 'AGENT_MESSAGE', 2); + */ +export function waitForEventCount( + threadManager: ThreadManager, + threadId: string, + eventType: LaceEventType, + count: number, + timeoutMs = 5000 +): Promise<LaceEvent[]> { + return new Promise((resolve, reject) => { + const startTime = Date.now(); + + const check = () => { + const events = threadManager.getEvents(threadId); + const matchingEvents = events.filter((e) => e.type === eventType); + + if (matchingEvents.length >= count) { + resolve(matchingEvents); + } else if (Date.now() - startTime > timeoutMs) { + reject( + new Error( + `Timeout waiting for ${count} ${eventType} events after ${timeoutMs}ms (got ${matchingEvents.length})` + ) + ); + } else { + setTimeout(check, 10); + } + }; + + check(); + }); +} + +/** + * Wait for an event matching a custom predicate + * Useful when you need to check event data, not just type + * + * @param threadManager - The thread manager to query + * @param threadId - Thread to check for events + * @param predicate - Function that returns true when event matches + * @param description - Human-readable description for error messages + * @param timeoutMs - Maximum time to wait (default 5000ms) + * @returns Promise resolving to the first matching event + * + * Example: + * // Wait for TOOL_RESULT with specific ID + * await waitForEventMatch( + * threadManager, + * agentThreadId, + * (e) => e.type === 'TOOL_RESULT' && e.data.id === 'call_123', + * 'TOOL_RESULT with id=call_123' + * ); + */ +export function waitForEventMatch( + threadManager: ThreadManager, + threadId: string, + predicate: (event: LaceEvent) => boolean, + description: string, + timeoutMs = 5000 +): Promise<LaceEvent> { + return new Promise((resolve, reject) => { + const startTime = Date.now(); + + const check = () => { + const events = threadManager.getEvents(threadId); + const event = events.find(predicate); + + if (event) { + resolve(event); + } else if (Date.now() - startTime > timeoutMs) { + reject(new Error(`Timeout waiting for ${description} after ${timeoutMs}ms`)); + } else { + setTimeout(check, 10); + } + }; + + check(); + }); +} + +// Usage example from actual debugging session: +// +// BEFORE (flaky): +// --------------- +// const messagePromise = agent.sendMessage('Execute tools'); +// await new Promise(r => setTimeout(r, 300)); // Hope tools start in 300ms +// agent.abort(); +// await messagePromise; +// await new Promise(r => setTimeout(r, 50)); // Hope results arrive in 50ms +// expect(toolResults.length).toBe(2); // Fails randomly +// +// AFTER (reliable): +// ---------------- +// const messagePromise = agent.sendMessage('Execute tools'); +// await waitForEventCount(threadManager, threadId, 'TOOL_CALL', 2); // Wait for tools to start +// agent.abort(); +// await messagePromise; +// await waitForEventCount(threadManager, threadId, 'TOOL_RESULT', 2); // Wait for results +// expect(toolResults.length).toBe(2); // Always succeeds +// +// Result: 60% pass rate → 100%, 40% faster execution diff --git a/.claude/skills/systematic-debugging/condition-based-waiting.md b/.claude/skills/systematic-debugging/condition-based-waiting.md new file mode 100644 index 0000000..70994f7 --- /dev/null +++ b/.claude/skills/systematic-debugging/condition-based-waiting.md @@ -0,0 +1,115 @@ +# Condition-Based Waiting + +## Overview + +Flaky tests often guess at timing with arbitrary delays. This creates race conditions where tests pass on fast machines but fail under load or in CI. + +**Core principle:** Wait for the actual condition you care about, not a guess about how long it takes. + +## When to Use + +```dot +digraph when_to_use { + "Test uses setTimeout/sleep?" [shape=diamond]; + "Testing timing behavior?" [shape=diamond]; + "Document WHY timeout needed" [shape=box]; + "Use condition-based waiting" [shape=box]; + + "Test uses setTimeout/sleep?" -> "Testing timing behavior?" [label="yes"]; + "Testing timing behavior?" -> "Document WHY timeout needed" [label="yes"]; + "Testing timing behavior?" -> "Use condition-based waiting" [label="no"]; +} +``` + +**Use when:** +- Tests have arbitrary delays (`setTimeout`, `sleep`, `time.sleep()`) +- Tests are flaky (pass sometimes, fail under load) +- Tests timeout when run in parallel +- Waiting for async operations to complete + +**Don't use when:** +- Testing actual timing behavior (debounce, throttle intervals) +- Always document WHY if using arbitrary timeout + +## Core Pattern + +```typescript +// ❌ BEFORE: Guessing at timing +await new Promise(r => setTimeout(r, 50)); +const result = getResult(); +expect(result).toBeDefined(); + +// ✅ AFTER: Waiting for condition +await waitFor(() => getResult() !== undefined); +const result = getResult(); +expect(result).toBeDefined(); +``` + +## Quick Patterns + +| Scenario | Pattern | +|----------|---------| +| Wait for event | `waitFor(() => events.find(e => e.type === 'DONE'))` | +| Wait for state | `waitFor(() => machine.state === 'ready')` | +| Wait for count | `waitFor(() => items.length >= 5)` | +| Wait for file | `waitFor(() => fs.existsSync(path))` | +| Complex condition | `waitFor(() => obj.ready && obj.value > 10)` | + +## Implementation + +Generic polling function: +```typescript +async function waitFor<T>( + condition: () => T | undefined | null | false, + description: string, + timeoutMs = 5000 +): Promise<T> { + const startTime = Date.now(); + + while (true) { + const result = condition(); + if (result) return result; + + if (Date.now() - startTime > timeoutMs) { + throw new Error(`Timeout waiting for ${description} after ${timeoutMs}ms`); + } + + await new Promise(r => setTimeout(r, 10)); // Poll every 10ms + } +} +``` + +See `condition-based-waiting-example.ts` in this directory for complete implementation with domain-specific helpers (`waitForEvent`, `waitForEventCount`, `waitForEventMatch`) from actual debugging session. + +## Common Mistakes + +**❌ Polling too fast:** `setTimeout(check, 1)` - wastes CPU +**✅ Fix:** Poll every 10ms + +**❌ No timeout:** Loop forever if condition never met +**✅ Fix:** Always include timeout with clear error + +**❌ Stale data:** Cache state before loop +**✅ Fix:** Call getter inside loop for fresh data + +## When Arbitrary Timeout IS Correct + +```typescript +// Tool ticks every 100ms - need 2 ticks to verify partial output +await waitForEvent(manager, 'TOOL_STARTED'); // First: wait for condition +await new Promise(r => setTimeout(r, 200)); // Then: wait for timed behavior +// 200ms = 2 ticks at 100ms intervals - documented and justified +``` + +**Requirements:** +1. First wait for triggering condition +2. Based on known timing (not guessing) +3. Comment explaining WHY + +## Real-World Impact + +From debugging session (2025-10-03): +- Fixed 15 flaky tests across 3 files +- Pass rate: 60% → 100% +- Execution time: 40% faster +- No more race conditions diff --git a/.claude/skills/systematic-debugging/defense-in-depth.md b/.claude/skills/systematic-debugging/defense-in-depth.md new file mode 100644 index 0000000..e248335 --- /dev/null +++ b/.claude/skills/systematic-debugging/defense-in-depth.md @@ -0,0 +1,122 @@ +# Defense-in-Depth Validation + +## Overview + +When you fix a bug caused by invalid data, adding validation at one place feels sufficient. But that single check can be bypassed by different code paths, refactoring, or mocks. + +**Core principle:** Validate at EVERY layer data passes through. Make the bug structurally impossible. + +## Why Multiple Layers + +Single validation: "We fixed the bug" +Multiple layers: "We made the bug impossible" + +Different layers catch different cases: +- Entry validation catches most bugs +- Business logic catches edge cases +- Environment guards prevent context-specific dangers +- Debug logging helps when other layers fail + +## The Four Layers + +### Layer 1: Entry Point Validation +**Purpose:** Reject obviously invalid input at API boundary + +```typescript +function createProject(name: string, workingDirectory: string) { + if (!workingDirectory || workingDirectory.trim() === '') { + throw new Error('workingDirectory cannot be empty'); + } + if (!existsSync(workingDirectory)) { + throw new Error(`workingDirectory does not exist: ${workingDirectory}`); + } + if (!statSync(workingDirectory).isDirectory()) { + throw new Error(`workingDirectory is not a directory: ${workingDirectory}`); + } + // ... proceed +} +``` + +### Layer 2: Business Logic Validation +**Purpose:** Ensure data makes sense for this operation + +```typescript +function initializeWorkspace(projectDir: string, sessionId: string) { + if (!projectDir) { + throw new Error('projectDir required for workspace initialization'); + } + // ... proceed +} +``` + +### Layer 3: Environment Guards +**Purpose:** Prevent dangerous operations in specific contexts + +```typescript +async function gitInit(directory: string) { + // In tests, refuse git init outside temp directories + if (process.env.NODE_ENV === 'test') { + const normalized = normalize(resolve(directory)); + const tmpDir = normalize(resolve(tmpdir())); + + if (!normalized.startsWith(tmpDir)) { + throw new Error( + `Refusing git init outside temp dir during tests: ${directory}` + ); + } + } + // ... proceed +} +``` + +### Layer 4: Debug Instrumentation +**Purpose:** Capture context for forensics + +```typescript +async function gitInit(directory: string) { + const stack = new Error().stack; + logger.debug('About to git init', { + directory, + cwd: process.cwd(), + stack, + }); + // ... proceed +} +``` + +## Applying the Pattern + +When you find a bug: + +1. **Trace the data flow** - Where does bad value originate? Where used? +2. **Map all checkpoints** - List every point data passes through +3. **Add validation at each layer** - Entry, business, environment, debug +4. **Test each layer** - Try to bypass layer 1, verify layer 2 catches it + +## Example from Session + +Bug: Empty `projectDir` caused `git init` in source code + +**Data flow:** +1. Test setup → empty string +2. `Project.create(name, '')` +3. `WorkspaceManager.createWorkspace('')` +4. `git init` runs in `process.cwd()` + +**Four layers added:** +- Layer 1: `Project.create()` validates not empty/exists/writable +- Layer 2: `WorkspaceManager` validates projectDir not empty +- Layer 3: `WorktreeManager` refuses git init outside tmpdir in tests +- Layer 4: Stack trace logging before git init + +**Result:** All 1847 tests passed, bug impossible to reproduce + +## Key Insight + +All four layers were necessary. During testing, each layer caught bugs the others missed: +- Different code paths bypassed entry validation +- Mocks bypassed business logic checks +- Edge cases on different platforms needed environment guards +- Debug logging identified structural misuse + +**Don't stop at one validation point.** Add checks at every layer. diff --git a/.claude/skills/systematic-debugging/find-polluter.sh b/.claude/skills/systematic-debugging/find-polluter.sh new file mode 100644 index 0000000..1d71c56 --- /dev/null +++ b/.claude/skills/systematic-debugging/find-polluter.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# Bisection script to find which test creates unwanted files/state +# Usage: ./find-polluter.sh <file_or_dir_to_check> <test_pattern> +# Example: ./find-polluter.sh '.git' 'src/**/*.test.ts' + +set -e + +if [ $# -ne 2 ]; then + echo "Usage: $0 <file_to_check> <test_pattern>" + echo "Example: $0 '.git' 'src/**/*.test.ts'" + exit 1 +fi + +POLLUTION_CHECK="$1" +TEST_PATTERN="$2" + +echo "🔍 Searching for test that creates: $POLLUTION_CHECK" +echo "Test pattern: $TEST_PATTERN" +echo "" + +# Get list of test files +TEST_FILES=$(find . -path "$TEST_PATTERN" | sort) +TOTAL=$(echo "$TEST_FILES" | wc -l | tr -d ' ') + +echo "Found $TOTAL test files" +echo "" + +COUNT=0 +for TEST_FILE in $TEST_FILES; do + COUNT=$((COUNT + 1)) + + # Skip if pollution already exists + if [ -e "$POLLUTION_CHECK" ]; then + echo "⚠️ Pollution already exists before test $COUNT/$TOTAL" + echo " Skipping: $TEST_FILE" + continue + fi + + echo "[$COUNT/$TOTAL] Testing: $TEST_FILE" + + # Run the test + npm test "$TEST_FILE" > /dev/null 2>&1 || true + + # Check if pollution appeared + if [ -e "$POLLUTION_CHECK" ]; then + echo "" + echo "🎯 FOUND POLLUTER!" + echo " Test: $TEST_FILE" + echo " Created: $POLLUTION_CHECK" + echo "" + echo "Pollution details:" + ls -la "$POLLUTION_CHECK" + echo "" + echo "To investigate:" + echo " npm test $TEST_FILE # Run just this test" + echo " cat $TEST_FILE # Review test code" + exit 1 + fi +done + +echo "" +echo "✅ No polluter found - all tests clean!" +exit 0 diff --git a/.claude/skills/systematic-debugging/root-cause-tracing.md b/.claude/skills/systematic-debugging/root-cause-tracing.md new file mode 100644 index 0000000..12ef522 --- /dev/null +++ b/.claude/skills/systematic-debugging/root-cause-tracing.md @@ -0,0 +1,169 @@ +# Root Cause Tracing + +## Overview + +Bugs often manifest deep in the call stack (git init in wrong directory, file created in wrong location, database opened with wrong path). Your instinct is to fix where the error appears, but that's treating a symptom. + +**Core principle:** Trace backward through the call chain until you find the original trigger, then fix at the source. + +## When to Use + +```dot +digraph when_to_use { + "Bug appears deep in stack?" [shape=diamond]; + "Can trace backwards?" [shape=diamond]; + "Fix at symptom point" [shape=box]; + "Trace to original trigger" [shape=box]; + "BETTER: Also add defense-in-depth" [shape=box]; + + "Bug appears deep in stack?" -> "Can trace backwards?" [label="yes"]; + "Can trace backwards?" -> "Trace to original trigger" [label="yes"]; + "Can trace backwards?" -> "Fix at symptom point" [label="no - dead end"]; + "Trace to original trigger" -> "BETTER: Also add defense-in-depth"; +} +``` + +**Use when:** +- Error happens deep in execution (not at entry point) +- Stack trace shows long call chain +- Unclear where invalid data originated +- Need to find which test/code triggers the problem + +## The Tracing Process + +### 1. Observe the Symptom +``` +Error: git init failed in ~/project/packages/core +``` + +### 2. Find Immediate Cause +**What code directly causes this?** +```typescript +await execFileAsync('git', ['init'], { cwd: projectDir }); +``` + +### 3. Ask: What Called This? +```typescript +WorktreeManager.createSessionWorktree(projectDir, sessionId) + → called by Session.initializeWorkspace() + → called by Session.create() + → called by test at Project.create() +``` + +### 4. Keep Tracing Up +**What value was passed?** +- `projectDir = ''` (empty string!) +- Empty string as `cwd` resolves to `process.cwd()` +- That's the source code directory! + +### 5. Find Original Trigger +**Where did empty string come from?** +```typescript +const context = setupCoreTest(); // Returns { tempDir: '' } +Project.create('name', context.tempDir); // Accessed before beforeEach! +``` + +## Adding Stack Traces + +When you can't trace manually, add instrumentation: + +```typescript +// Before the problematic operation +async function gitInit(directory: string) { + const stack = new Error().stack; + console.error('DEBUG git init:', { + directory, + cwd: process.cwd(), + nodeEnv: process.env.NODE_ENV, + stack, + }); + + await execFileAsync('git', ['init'], { cwd: directory }); +} +``` + +**Critical:** Use `console.error()` in tests (not logger - may not show) + +**Run and capture:** +```bash +npm test 2>&1 | grep 'DEBUG git init' +``` + +**Analyze stack traces:** +- Look for test file names +- Find the line number triggering the call +- Identify the pattern (same test? same parameter?) + +## Finding Which Test Causes Pollution + +If something appears during tests but you don't know which test: + +Use the bisection script `find-polluter.sh` in this directory: + +```bash +./find-polluter.sh '.git' 'src/**/*.test.ts' +``` + +Runs tests one-by-one, stops at first polluter. See script for usage. + +## Real Example: Empty projectDir + +**Symptom:** `.git` created in `packages/core/` (source code) + +**Trace chain:** +1. `git init` runs in `process.cwd()` ← empty cwd parameter +2. WorktreeManager called with empty projectDir +3. Session.create() passed empty string +4. Test accessed `context.tempDir` before beforeEach +5. setupCoreTest() returns `{ tempDir: '' }` initially + +**Root cause:** Top-level variable initialization accessing empty value + +**Fix:** Made tempDir a getter that throws if accessed before beforeEach + +**Also added defense-in-depth:** +- Layer 1: Project.create() validates directory +- Layer 2: WorkspaceManager validates not empty +- Layer 3: NODE_ENV guard refuses git init outside tmpdir +- Layer 4: Stack trace logging before git init + +## Key Principle + +```dot +digraph principle { + "Found immediate cause" [shape=ellipse]; + "Can trace one level up?" [shape=diamond]; + "Trace backwards" [shape=box]; + "Is this the source?" [shape=diamond]; + "Fix at source" [shape=box]; + "Add validation at each layer" [shape=box]; + "Bug impossible" [shape=doublecircle]; + "NEVER fix just the symptom" [shape=octagon, style=filled, fillcolor=red, fontcolor=white]; + + "Found immediate cause" -> "Can trace one level up?"; + "Can trace one level up?" -> "Trace backwards" [label="yes"]; + "Can trace one level up?" -> "NEVER fix just the symptom" [label="no"]; + "Trace backwards" -> "Is this the source?"; + "Is this the source?" -> "Trace backwards" [label="no - keeps going"]; + "Is this the source?" -> "Fix at source" [label="yes"]; + "Fix at source" -> "Add validation at each layer"; + "Add validation at each layer" -> "Bug impossible"; +} +``` + +**NEVER fix just where the error appears.** Trace back to find the original trigger. + +## Stack Trace Tips + +**In tests:** Use `console.error()` not logger - logger may be suppressed +**Before operation:** Log before the dangerous operation, not after it fails +**Include context:** Directory, cwd, environment variables, timestamps +**Capture stack:** `new Error().stack` shows complete call chain + +## Real-World Impact + +From debugging session (2025-10-03): +- Found root cause through 5-level trace +- Fixed at source (getter validation) +- Added 4 layers of defense +- 1847 tests passed, zero pollution diff --git a/.claude/skills/test-driven-development/SKILL.md b/.claude/skills/test-driven-development/SKILL.md new file mode 100644 index 0000000..8b9d1d8 --- /dev/null +++ b/.claude/skills/test-driven-development/SKILL.md @@ -0,0 +1,43 @@ +--- +name: test-driven-development +description: "Testing outcome gate for features and bugfixes: before any merge, every behavior change must be backed by test evidence that failed before the change and passes after it. Defines what counts as evidence and what test quality means; does not prescribe your coding order." +--- + +# 测试结果门(Test Evidence Gate) + +## 结果门(合并前的硬性验收) + +``` +每个行为变更,合并前必须有"先失败、后通过"的测试证据。 +``` + +具体要求: + +1. **红→绿证据**: 对每个新行为/修复,存在一个测试,你能出示它在变更前失败(或对着被还原的代码失败)、在变更后通过的**本会话工具输出**。测试从未失败过 = 无法证明它在测真东西。 +2. **Bug 修复必带回归测试**: 先有复现该 bug 的失败测试,再谈修复。修完后该测试通过、其余测试不回归。 +3. **全绿交付**: 合并前全部测试通过,输出干净(无报错、无告警噪音)。 + +为什么要求"先失败":变更后才补写的测试天然通过,它验证的是"代码做了什么"而不是"代码应该做什么"——你没见过它抓到问题,就不知道它能不能抓到问题。事后补测的情况下,用"还原变更→测试转红→恢复变更→测试转绿"的方式补出证据。 + +写代码的顺序(先测后码、先码后测、边探索边写)不做规定;探索性代码写完后,仍要按上述方式补出红绿证据才能合并。 + +**例外**(与人类确认后): 一次性原型、生成代码、纯配置文件。 + +## 测试质量(证据必须是真证据) + +| 要求 | 说明 | +|---|---| +| 测行为,不测 mock | 测试对象是真实代码路径;mock 只用于不可控外部依赖(真实网关、计费 API)。断言 mock 被调用了几次 ≠ 测试了行为 | +| 一个测试一个行为 | 名字里需要"和"就拆开;名字描述行为而非编号 | +| 真实样本优先 | 用录制的真实网关响应或其二次构造,不凭空捏造响应结构(CLAUDE.md §4.6) | +| 并发/韧性是一等对象 | 重试穿透取消、熔断开路半开、限流结算退款、降级方向都要有测试 | + +添加 mock 或测试工具前,读 `testing-anti-patterns.md`(测 mock 行为、给生产类加测试专用方法等常见坑)。 + +## 常见自欺(仍然不算证据) + +- "太简单不用测" —— 简单代码也会坏;库的 bug 击穿所有下游。 +- "我手动验证过了" —— 无记录、不可重跑,不是证据。 +- "测试一次通过,应该没问题" —— 从未失败过的测试,不知道它测的是什么。 + +运行命令统一 `conda run -n PolyGateway pytest ...`。 diff --git a/.claude/skills/test-driven-development/testing-anti-patterns.md b/.claude/skills/test-driven-development/testing-anti-patterns.md new file mode 100644 index 0000000..e77ab6b --- /dev/null +++ b/.claude/skills/test-driven-development/testing-anti-patterns.md @@ -0,0 +1,299 @@ +# Testing Anti-Patterns + +**Load this reference when:** writing or changing tests, adding mocks, or tempted to add test-only methods to production code. + +## Overview + +Tests must verify real behavior, not mock behavior. Mocks are a means to isolate, not the thing being tested. + +**Core principle:** Test what the code does, not what the mocks do. + +**Following strict TDD prevents these anti-patterns.** + +## The Iron Laws + +``` +1. NEVER test mock behavior +2. NEVER add test-only methods to production classes +3. NEVER mock without understanding dependencies +``` + +## Anti-Pattern 1: Testing Mock Behavior + +**The violation:** +```typescript +// ❌ BAD: Testing that the mock exists +test('renders sidebar', () => { + render(<Page />); + expect(screen.getByTestId('sidebar-mock')).toBeInTheDocument(); +}); +``` + +**Why this is wrong:** +- You're verifying the mock works, not that the component works +- Test passes when mock is present, fails when it's not +- Tells you nothing about real behavior + +**your human partner's correction:** "Are we testing the behavior of a mock?" + +**The fix:** +```typescript +// ✅ GOOD: Test real component or don't mock it +test('renders sidebar', () => { + render(<Page />); // Don't mock sidebar + expect(screen.getByRole('navigation')).toBeInTheDocument(); +}); + +// OR if sidebar must be mocked for isolation: +// Don't assert on the mock - test Page's behavior with sidebar present +``` + +### Gate Function + +``` +BEFORE asserting on any mock element: + Ask: "Am I testing real component behavior or just mock existence?" + + IF testing mock existence: + STOP - Delete the assertion or unmock the component + + Test real behavior instead +``` + +## Anti-Pattern 2: Test-Only Methods in Production + +**The violation:** +```typescript +// ❌ BAD: destroy() only used in tests +class Session { + async destroy() { // Looks like production API! + await this._workspaceManager?.destroyWorkspace(this.id); + // ... cleanup + } +} + +// In tests +afterEach(() => session.destroy()); +``` + +**Why this is wrong:** +- Production class polluted with test-only code +- Dangerous if accidentally called in production +- Violates YAGNI and separation of concerns +- Confuses object lifecycle with entity lifecycle + +**The fix:** +```typescript +// ✅ GOOD: Test utilities handle test cleanup +// Session has no destroy() - it's stateless in production + +// In test-utils/ +export async function cleanupSession(session: Session) { + const workspace = session.getWorkspaceInfo(); + if (workspace) { + await workspaceManager.destroyWorkspace(workspace.id); + } +} + +// In tests +afterEach(() => cleanupSession(session)); +``` + +### Gate Function + +``` +BEFORE adding any method to production class: + Ask: "Is this only used by tests?" + + IF yes: + STOP - Don't add it + Put it in test utilities instead + + Ask: "Does this class own this resource's lifecycle?" + + IF no: + STOP - Wrong class for this method +``` + +## Anti-Pattern 3: Mocking Without Understanding + +**The violation:** +```typescript +// ❌ BAD: Mock breaks test logic +test('detects duplicate server', () => { + // Mock prevents config write that test depends on! + vi.mock('ToolCatalog', () => ({ + discoverAndCacheTools: vi.fn().mockResolvedValue(undefined) + })); + + await addServer(config); + await addServer(config); // Should throw - but won't! +}); +``` + +**Why this is wrong:** +- Mocked method had side effect test depended on (writing config) +- Over-mocking to "be safe" breaks actual behavior +- Test passes for wrong reason or fails mysteriously + +**The fix:** +```typescript +// ✅ GOOD: Mock at correct level +test('detects duplicate server', () => { + // Mock the slow part, preserve behavior test needs + vi.mock('MCPServerManager'); // Just mock slow server startup + + await addServer(config); // Config written + await addServer(config); // Duplicate detected ✓ +}); +``` + +### Gate Function + +``` +BEFORE mocking any method: + STOP - Don't mock yet + + 1. Ask: "What side effects does the real method have?" + 2. Ask: "Does this test depend on any of those side effects?" + 3. Ask: "Do I fully understand what this test needs?" + + IF depends on side effects: + Mock at lower level (the actual slow/external operation) + OR use test doubles that preserve necessary behavior + NOT the high-level method the test depends on + + IF unsure what test depends on: + Run test with real implementation FIRST + Observe what actually needs to happen + THEN add minimal mocking at the right level + + Red flags: + - "I'll mock this to be safe" + - "This might be slow, better mock it" + - Mocking without understanding the dependency chain +``` + +## Anti-Pattern 4: Incomplete Mocks + +**The violation:** +```typescript +// ❌ BAD: Partial mock - only fields you think you need +const mockResponse = { + status: 'success', + data: { userId: '123', name: 'Alice' } + // Missing: metadata that downstream code uses +}; + +// Later: breaks when code accesses response.metadata.requestId +``` + +**Why this is wrong:** +- **Partial mocks hide structural assumptions** - You only mocked fields you know about +- **Downstream code may depend on fields you didn't include** - Silent failures +- **Tests pass but integration fails** - Mock incomplete, real API complete +- **False confidence** - Test proves nothing about real behavior + +**The Iron Rule:** Mock the COMPLETE data structure as it exists in reality, not just fields your immediate test uses. + +**The fix:** +```typescript +// ✅ GOOD: Mirror real API completeness +const mockResponse = { + status: 'success', + data: { userId: '123', name: 'Alice' }, + metadata: { requestId: 'req-789', timestamp: 1234567890 } + // All fields real API returns +}; +``` + +### Gate Function + +``` +BEFORE creating mock responses: + Check: "What fields does the real API response contain?" + + Actions: + 1. Examine actual API response from docs/examples + 2. Include ALL fields system might consume downstream + 3. Verify mock matches real response schema completely + + Critical: + If you're creating a mock, you must understand the ENTIRE structure + Partial mocks fail silently when code depends on omitted fields + + If uncertain: Include all documented fields +``` + +## Anti-Pattern 5: Integration Tests as Afterthought + +**The violation:** +``` +✅ Implementation complete +❌ No tests written +"Ready for testing" +``` + +**Why this is wrong:** +- Testing is part of implementation, not optional follow-up +- TDD would have caught this +- Can't claim complete without tests + +**The fix:** +``` +TDD cycle: +1. Write failing test +2. Implement to pass +3. Refactor +4. THEN claim complete +``` + +## When Mocks Become Too Complex + +**Warning signs:** +- Mock setup longer than test logic +- Mocking everything to make test pass +- Mocks missing methods real components have +- Test breaks when mock changes + +**your human partner's question:** "Do we need to be using a mock here?" + +**Consider:** Integration tests with real components often simpler than complex mocks + +## TDD Prevents These Anti-Patterns + +**Why TDD helps:** +1. **Write test first** → Forces you to think about what you're actually testing +2. **Watch it fail** → Confirms test tests real behavior, not mocks +3. **Minimal implementation** → No test-only methods creep in +4. **Real dependencies** → You see what the test actually needs before mocking + +**If you're testing mock behavior, you violated TDD** - you added mocks without watching test fail against real code first. + +## Quick Reference + +| Anti-Pattern | Fix | +|--------------|-----| +| Assert on mock elements | Test real component or unmock it | +| Test-only methods in production | Move to test utilities | +| Mock without understanding | Understand dependencies first, mock minimally | +| Incomplete mocks | Mirror real API completely | +| Tests as afterthought | TDD - tests first | +| Over-complex mocks | Consider integration tests | + +## Red Flags + +- Assertion checks for `*-mock` test IDs +- Methods only called in test files +- Mock setup is >50% of test +- Test fails when you remove mock +- Can't explain why mock is needed +- Mocking "just to be safe" + +## The Bottom Line + +**Mocks are tools to isolate, not things to test.** + +If TDD reveals you're testing mock behavior, you've gone wrong. + +Fix: Test real behavior or question why you're mocking at all. diff --git a/.claude/skills/using-git-worktrees/SKILL.md b/.claude/skills/using-git-worktrees/SKILL.md new file mode 100644 index 0000000..e8c7d01 --- /dev/null +++ b/.claude/skills/using-git-worktrees/SKILL.md @@ -0,0 +1,214 @@ +--- +name: using-git-worktrees +description: Use when starting feature work that needs isolation from current workspace or before executing implementation plans - ensures an isolated workspace exists via native tools or git worktree fallback +--- + +# Using Git Worktrees + +## Overview + +Ensure work happens in an isolated workspace. Prefer your platform's native worktree tools. Fall back to manual git worktrees only when no native tool is available. + +**Core principle:** Detect existing isolation first. Then use native tools. Then fall back to git. Never fight the harness. + +**Announce at start:** "I'm using the using-git-worktrees skill to set up an isolated workspace." + +## Step 0: Detect Existing Isolation + +**Before creating anything, check if you are already in an isolated workspace.** + +```bash +GIT_DIR=$(cd "$(git rev-parse --git-dir)" 2>/dev/null && pwd -P) +GIT_COMMON=$(cd "$(git rev-parse --git-common-dir)" 2>/dev/null && pwd -P) +BRANCH=$(git branch --show-current) +``` + +**Submodule guard:** `GIT_DIR != GIT_COMMON` is also true inside git submodules. Before concluding "already in a worktree," verify you are not in a submodule: + +```bash +# If this returns a path, you're in a submodule, not a worktree — treat as normal repo +git rev-parse --show-superproject-working-tree 2>/dev/null +``` + +**If `GIT_DIR != GIT_COMMON` (and not a submodule):** You are already in a linked worktree. Skip to Step 3 (Project Setup). Do NOT create another worktree. + +Report with branch state: +- On a branch: "Already in isolated workspace at `<path>` on branch `<name>`." +- Detached HEAD: "Already in isolated workspace at `<path>` (detached HEAD, externally managed). Branch creation needed at finish time." + +**If `GIT_DIR == GIT_COMMON` (or in a submodule):** You are in a normal repo checkout. + +Has the user already indicated their worktree preference in your instructions? If not, ask for consent before creating a worktree: + +> "Would you like me to set up an isolated worktree? It protects your current branch from changes." + +Honor any existing declared preference without asking. If the user declines consent, work in place and skip to Step 3. + +## Step 1: Create Isolated Workspace + +**You have two mechanisms. Try them in this order.** + +### 1a. Native Worktree Tools (preferred) + +The user has asked for an isolated workspace (Step 0 consent). Do you already have a way to create a worktree? It might be a tool with a name like `EnterWorktree`, `WorktreeCreate`, a `/worktree` command, or a `--worktree` flag. If you do, use it and skip to Step 3. + +Native tools handle directory placement, branch creation, and cleanup automatically. Using `git worktree add` when you have a native tool creates phantom state your harness can't see or manage. + +Only proceed to Step 1b if you have no native worktree tool available. + +### 1b. Git Worktree Fallback + +**Only use this if Step 1a does not apply** — you have no native worktree tool available. Create a worktree manually using git. + +#### Directory Selection + +Follow this priority order. Explicit user preference always beats observed filesystem state. + +1. **Check your instructions for a declared worktree directory preference.** If the user has already specified one, use it without asking. + +2. **Check for an existing project-local worktree directory:** + ```bash + ls -d .worktrees 2>/dev/null # Preferred (hidden) + ls -d worktrees 2>/dev/null # Alternative + ``` + If found, use it. If both exist, `.worktrees` wins. + +3. **Check for an existing global directory:** + ```bash + project=$(basename "$(git rev-parse --show-toplevel)") + ls -d ~/.config/superpowers/worktrees/$project 2>/dev/null + ``` + If found, use it (backward compatibility with legacy global path). + +4. **If there is no other guidance available**, default to `.worktrees/` at the project root. + +#### Safety Verification (project-local directories only) + +**MUST verify directory is ignored before creating worktree:** + +```bash +git check-ignore -q .worktrees 2>/dev/null || git check-ignore -q worktrees 2>/dev/null +``` + +**If NOT ignored:** Add to .gitignore, commit the change, then proceed. + +**Why critical:** Prevents accidentally committing worktree contents to repository. + +Global directories (`~/.config/superpowers/worktrees/`) need no verification. + +#### Create the Worktree + +```bash +project=$(basename "$(git rev-parse --show-toplevel)") + +# Determine path based on chosen location +# For project-local: path="$LOCATION/$BRANCH_NAME" +# For global: path="~/.config/superpowers/worktrees/$project/$BRANCH_NAME" + +git worktree add "$path" -b "$BRANCH_NAME" +cd "$path" +``` + +**Sandbox fallback:** If `git worktree add` fails with a permission error (sandbox denial), tell the user the sandbox blocked worktree creation and you're working in the current directory instead. Then run setup and baseline tests in place. + +## Step 3: Project Setup + +Auto-detect and run appropriate setup: + +```bash +# Node.js +if [ -f package.json ]; then npm install; fi + +# Rust +if [ -f Cargo.toml ]; then cargo build; fi + +# Python (PolyGateway conda env) +if [ -f requirements.txt ]; then conda run -n PolyGateway pip install -r requirements.txt; fi + +# Go +if [ -f go.mod ]; then go mod download; fi +``` + +## Step 4: Verify Clean Baseline + +Run tests to ensure workspace starts clean: + +```bash +# Use project-appropriate command +npm test / cargo test / conda run -n PolyGateway pytest / go test ./... +``` + +**If tests fail:** Report failures, ask whether to proceed or investigate. + +**If tests pass:** Report ready. + +### Report + +``` +Worktree ready at <full-path> +Tests passing (<N> tests, 0 failures) +Ready to implement <feature-name> +``` + +## Quick Reference + +| Situation | Action | +|-----------|--------| +| Already in linked worktree | Skip creation (Step 0) | +| In a submodule | Treat as normal repo (Step 0 guard) | +| Native worktree tool available | Use it (Step 1a) | +| No native tool | Git worktree fallback (Step 1b) | +| `.worktrees/` exists | Use it (verify ignored) | +| `worktrees/` exists | Use it (verify ignored) | +| Both exist | Use `.worktrees/` | +| Neither exists | Check instruction file, then default `.worktrees/` | +| Global path exists | Use it (backward compat) | +| Directory not ignored | Add to .gitignore + commit | +| Permission error on create | Sandbox fallback, work in place | +| Tests fail during baseline | Report failures + ask | +| No package.json/Cargo.toml | Skip dependency install | + +## Common Mistakes + +### Fighting the harness + +- **Problem:** Using `git worktree add` when the platform already provides isolation +- **Fix:** Step 0 detects existing isolation. Step 1a defers to native tools. + +### Skipping detection + +- **Problem:** Creating a nested worktree inside an existing one +- **Fix:** Always run Step 0 before creating anything + +### Skipping ignore verification + +- **Problem:** Worktree contents get tracked, pollute git status +- **Fix:** Always use `git check-ignore` before creating project-local worktree + +### Assuming directory location + +- **Problem:** Creates inconsistency, violates project conventions +- **Fix:** Follow priority: existing > global legacy > instruction file > default + +### Proceeding with failing tests + +- **Problem:** Can't distinguish new bugs from pre-existing issues +- **Fix:** Report failures, get explicit permission to proceed + +## Red Flags + +**Never:** +- Create a worktree when Step 0 detects existing isolation +- Use `git worktree add` when you have a native worktree tool (e.g., `EnterWorktree`). This is the #1 mistake — if you have it, use it. +- Skip Step 1a by jumping straight to Step 1b's git commands +- Create worktree without verifying it's ignored (project-local) +- Skip baseline test verification +- Proceed with failing tests without asking + +**Always:** +- Run Step 0 detection first +- Prefer native tools over git fallback +- Follow directory priority: existing > global legacy > instruction file > default +- Verify directory is ignored for project-local +- Auto-detect and run project setup +- Verify clean test baseline diff --git a/.claude/skills/verification-before-completion/SKILL.md b/.claude/skills/verification-before-completion/SKILL.md new file mode 100644 index 0000000..df906d6 --- /dev/null +++ b/.claude/skills/verification-before-completion/SKILL.md @@ -0,0 +1,56 @@ +--- +name: verification-before-completion +description: "Use before claiming work is complete, fixed, or passing, and before committing or creating PRs. Every completion claim must map to tool output produced in this session; for milestone-scale or multi-file work, dispatch a fresh-context verifier subagent instead of self-review." +--- + +# Verification Before Completion + +## 边界声明 + +``` +没有本会话的新鲜验证证据,就不得声称完成。 +``` + +"应该可以了"、"看起来没问题"、"我有信心"都不是证据。证据 = 本会话中实际运行的命令及其输出。 + +## 证据化声明 + +最终报告里的每一条"已完成 / 已修复 / 已通过",都必须能对应到本会话的一次工具结果: + +| 声明 | 所需证据 | +|---|---| +| 测试通过 | 本会话 `conda run -n PolyGateway pytest` 输出:0 failures | +| lint/格式干净 | `make lint` / `ruff check` 输出:0 errors | +| bug 已修复 | 复现原症状的测试:变更前失败、变更后通过(见 `test-driven-development` 结果门) | +| 需求已满足 | 对照 plan/spec 逐条核对的清单,每条指向 diff 或测试 | +| subagent 完成了任务 | 你亲自查看的 `git diff` / 文件内容,而非 subagent 的自述 | +| 无残留半成品 | grep TODO/NotImplementedError/占位 mock:0 命中;工作区无临时脚本与调试产物 | + +无法出示证据的声明,改为陈述实际状态("X 已实现,Y 尚未验证,因为…")。 + +## 独立验证(里程碑级/多文件工作必做) + +自我检查会继承自己的盲区。达到以下任一条件时,**必须派一个全新上下文的 verifier subagent**(只读)做独立核验,而不是自己过检查清单: + +- 里程碑级任务或跨多文件的功能完成时; +- 提 PR / 合并回主线前; +- 执行完一份 plan 时。 + +做法:用 `Agent` 工具派一个 read-only subagent(如 `Explore` 或 general-purpose 限只读),prompt 只给它:任务需求原文(spec/plan 相关部分)、变更范围(分支/SHA 区间)、验证命令。**不给它你的实现思路与自评**——它的价值就在于没有你的上下文。要求它: + +1. 亲自读 diff 与相关文件,逐条核对需求覆盖(缺失/多余/误解); +2. 亲自跑测试与 lint,报告真实输出; +3. 按 Critical/Important/Minor 返回问题清单。 + +对其结论逐条核验后修复;Critical/Important 未清零不得声称完成。小改动(单文件、明确修复)不必派 subagent,但证据化声明的要求不变。 + +## 留痕(`research-wiki/` 存在时) + +验证**发现问题**时(全过则不写),记一条 finding: + +```bash +.claude/tools/research_wiki.py add_entity research-wiki/ --type finding --id <slug> --title "<验证问题>" +.claude/tools/research_wiki.py rebuild_index research-wiki/ +``` + +页内记录:验证命令、实际输出、失败原因、修复后结果。 diff --git a/.claude/skills/writing-plans/SKILL.md b/.claude/skills/writing-plans/SKILL.md new file mode 100644 index 0000000..8c506d3 --- /dev/null +++ b/.claude/skills/writing-plans/SKILL.md @@ -0,0 +1,65 @@ +--- +name: writing-plans +description: "Use for milestone-scale or multi-file feature work: write an implementation plan before coding. MANDATORY for work spanning multiple modules or implementing an approved design. For small, well-bounded changes (single file, clear fix), plan inline and skip this skill." +--- + +# Writing Plans + +## 触发边界 + +- **强制**: 里程碑级任务、跨多文件的新功能、实现一份已批准设计的工作。 +- **自判**: 单文件小改动、明确的 bug 修复、纯文档/配置变更——不值得为它们写计划文档。 + +计划保存至 `research-wiki/plans/YYYY-MM-DD-<feature-name>.md`(≤1000 行)。 + +## 计划要写给谁 + +假设执行者(可能是 subagent、可能是未来的你)对本代码库零上下文。计划要交代:每个任务动哪些文件(精确路径)、验收标准是什么、怎么验证。DRY、YAGNI、频繁提交。不写计划外的重构与抽象。 + +## 计划必备内容 + +**头部**: 目标(一句话)、方案概述(2-3 句)、涉及技术。 + +**文件结构**: 开始拆任务前,先列出将创建/修改的文件及各自职责——分解决策在这里锁定。遵循既有架构(`ports/types/errors` 内核 + middleware/transports/backends/telemetry,见 ARCHITECTURE.md §8)。 + +**任务清单**: 每个任务包含: +- 精确文件路径(创建/修改/测试); +- 要实现的行为与验收标准; +- 测试要求:该任务合并前必须能出示"先失败后通过"的测试证据(见 `test-driven-development` 的结果门); +- 验证命令及预期输出(如 `conda run -n PolyGateway pytest tests/... -v` → PASS); +- 提交点(checkbox `- [ ]` 语法便于追踪)。 + +任务粒度以"一次提交、独立可验证"为准,不必把每个动作拆成几分钟一格的脚本步骤——执行者会自己安排动作顺序。 + +## No Placeholders(计划失败模式,禁止出现) + +- "TBD" / "TODO" / "以后补充" / "实现细节略" +- "添加适当的错误处理 / 校验 / 边界处理"(不说清楚是什么) +- "为上文写测试"(没有说明测什么行为) +- "同 Task N"(执行者可能乱序读,关键内容要重复或明确引用) +- 引用了任何任务中都未定义的类型/函数/方法 + +关键接口(跨任务消费的类型、函数签名)必须在计划中写出实际代码;其余代码执行时再写。 + +## 保真校验(迁移类计划必做) + +本库的主体工作是把 `reference/` 三项目的治理代码迁移进库。若计划涉及 ARCHITECTURE.md §1.4"关键资产索引"中列出的任何移植蓝本(治理主循环、流式看门狗、Redis+Lua 限流、跨进程熔断、错误分类、遥测等): + +1. 在对应任务中标注参考文件路径,要求实现时**逐段比对参考实现**; +2. 为该任务加"保真校验"检查点:核心逻辑(条件分支、Lua 脚本语义、状态机转换、退避公式)不得被简化或悄悄改变行为——重构结构可以,改变语义必须在设计中声明过。 + +计划不涉及迁移时,注明"本计划不涉及参考实现迁移,保真校验不适用"。 + +## 审核门(保留;plan 无需人类批准) + +1. **自审**: 对照 spec 逐节检查覆盖(每条需求能指到任务)、扫 placeholder、检查跨任务类型/签名一致性。发现问题就地修。 +2. **Codex 审**: 用 `/codex:rescue --fresh --wait` 只读审查(设计覆盖、每步可执行性、测试证据要求是否齐全),模板见 `plan-document-reviewer-prompt.md`。意见仅供参考,逐条核验后自行决定采纳,就地修订。 +3. 通过后**直接进入执行**,无需人类批准(与 design 的人类门不同)。执行方式自选:任务多且相互独立的大计划可用 `subagent-driven-development`;中小计划直接按计划实现。 + +## Wiki 注册(`research-wiki/` 存在时) + +```bash +.claude/tools/research_wiki.py add_entity research-wiki/ --type plan --id <slug> --title "<title>" +.claude/tools/research_wiki.py add_edge research-wiki/ --from "plan:<id>" --to "design:<id>" --type implements --evidence "..." +.claude/tools/research_wiki.py rebuild_index research-wiki/ +``` diff --git a/.claude/skills/writing-plans/plan-document-reviewer-prompt.md b/.claude/skills/writing-plans/plan-document-reviewer-prompt.md new file mode 100644 index 0000000..2222052 --- /dev/null +++ b/.claude/skills/writing-plans/plan-document-reviewer-prompt.md @@ -0,0 +1,49 @@ +# Plan Document Reviewer Prompt Template + +Use this template when dispatching a plan document reviewer subagent. + +**Purpose:** Verify the plan is complete, matches the spec, and has proper task decomposition. + +**Dispatch after:** The complete plan is written. + +``` +Task tool (general-purpose): + description: "Review plan document" + prompt: | + You are a plan document reviewer. Verify this plan is complete and ready for implementation. + + **Plan to review:** [PLAN_FILE_PATH] + **Spec for reference:** [SPEC_FILE_PATH] + + ## What to Check + + | Category | What to Look For | + |----------|------------------| + | Completeness | TODOs, placeholders, incomplete tasks, missing steps | + | Spec Alignment | Plan covers spec requirements, no major scope creep | + | Task Decomposition | Tasks have clear boundaries, steps are actionable | + | Buildability | Could an engineer follow this plan without getting stuck? | + + ## Calibration + + **Only flag issues that would cause real problems during implementation.** + An implementer building the wrong thing or getting stuck is an issue. + Minor wording, stylistic preferences, and "nice to have" suggestions are not. + + Approve unless there are serious gaps — missing requirements from the spec, + contradictory steps, placeholder content, or tasks so vague they can't be acted on. + + ## Output Format + + ## Plan Review + + **Status:** Approved | Issues Found + + **Issues (if any):** + - [Task X, Step Y]: [specific issue] - [why it matters for implementation] + + **Recommendations (advisory, do not block approval):** + - [suggestions for improvement] +``` + +**Reviewer returns:** Status, Issues (if any), Recommendations \ No newline at end of file diff --git a/.claude/templates/EXPERIMENT_LOG_CN.md b/.claude/templates/EXPERIMENT_LOG_CN.md new file mode 100644 index 0000000..f2b9891 --- /dev/null +++ b/.claude/templates/EXPERIMENT_LOG_CN.md @@ -0,0 +1,49 @@ +# 实验记录模板 + +> 每次实验都建议独立记录，便于复现、回溯和汇总。 + +## 实验记录 + +### 名称 +[填写：实验名称] + +### 日期 +[填写：YYYY-MM-DD] + +### Idea +[填写：对应的 idea 名称或编号] + +### 目标 +[填写：这次实验要回答什么问题，验证哪个 claim。] + +## Setup + +| 项目 | 说明 | +|---|---| +| 方法 | [填写] | +| 数据集 | [填写] | +| 基线 | [填写] | +| 硬件 | [填写] | +| 配置 | [填写] | + +## 结果表格 + +| method | dataset | metric-1 | metric-2 | notes | +|---|---|---:|---:|---| +| [填写] | [填写] | [填写] | [填写] | [填写] | +| [填写] | [填写] | [填写] | [填写] | [填写] | + +## 结论 + +### 是否支持 claim? +[填写：支持 / 部分支持 / 不支持，并说明原因。] + +### 关键收获 +[填写：最重要的观察、异常现象、后续启发。] + +## 复现命令 + +```bash +[填写：完整复现命令] +``` + diff --git a/.claude/templates/EXPERIMENT_PLAN_CN.md b/.claude/templates/EXPERIMENT_PLAN_CN.md new file mode 100644 index 0000000..e4e75a6 --- /dev/null +++ b/.claude/templates/EXPERIMENT_PLAN_CN.md @@ -0,0 +1,49 @@ +# 实验计划模板 + +> 用于把 claim、实验块、运行顺序和资源预算拆成可执行计划。 + +## 问题 / 方法论点 + +[填写：本轮实验要验证的关键问题，以及方法主张是什么。] + +## Claim 映射表 + +| claim | 重要性 | 最低证据 | 关联实验块 | +|---|---|---|---| +| [填写] | [填写] | [填写] | [填写] | +| [填写] | [填写] | [填写] | [填写] | +| [填写] | [填写] | [填写] | [填写] | + +## 实验块 + +| 实验块 | 验证 claim | 数据集 | 对比系统 | 指标 | 成功标准 | 失败解读 | 优先级 | +|---|---|---|---|---|---|---|---| +| [填写] | [填写] | [填写] | [填写] | [填写] | [填写] | [填写] | [填写] | +| [填写] | [填写] | [填写] | [填写] | [填写] | [填写] | [填写] | [填写] | +| [填写] | [填写] | [填写] | [填写] | [填写] | [填写] | [填写] | [填写] | + +## 运行顺序表 + +| 里程碑 | 目标 | 运行内容 | 决策关卡 | 预估耗时 | +|---|---|---|---|---| +| [填写] | [填写] | [填写] | [填写] | [填写] | +| [填写] | [填写] | [填写] | [填写] | [填写] | +| [填写] | [填写] | [填写] | [填写] | [填写] | + +## 算力预算 + +| 项目 | 预算 | 备注 | +|---|---|---| +| 训练 | [填写] | [填写] | +| 评估 | [填写] | [填写] | +| 消融 | [填写] | [填写] | +| 总计 | [填写] | [填写] | + +## 风险 + +| 风险 | 影响 | 缓解措施 | +|---|---|---| +| [填写] | [填写] | [填写] | +| [填写] | [填写] | [填写] | +| [填写] | [填写] | [填写] | + diff --git a/.claude/templates/IDEA_CANDIDATES_CN.md b/.claude/templates/IDEA_CANDIDATES_CN.md new file mode 100644 index 0000000..c397e50 --- /dev/null +++ b/.claude/templates/IDEA_CANDIDATES_CN.md @@ -0,0 +1,30 @@ +# Idea 候选池模板 + +> 用于跟踪当前候选方案、淘汰原因和切换历史。 + +## 当前 Idea + +[填写：当前正在推进的 idea 名称、核心假设和一句话概述。] + +## 候选列表 + +| Idea 名称 | 一句话描述 | 新颖性评分 (X/10) | 评审意见 | 试点结果 | 预估工作量 | 未优先选择原因 | +|---|---|---:|---|---|---|---| +| [填写] | [填写] | [填写] | [填写] | [填写] | [填写] | [填写] | +| [填写] | [填写] | [填写] | [填写] | [填写] | [填写] | [填写] | +| [填写] | [填写] | [填写] | [填写] | [填写] | [填写] | [填写] | + +## 已淘汰 Idea + +| Idea 名称 | 淘汰原因 | 日期 | 来源 | +|---|---|---|---| +| [填写] | [填写] | [填写] | [填写] | +| [填写] | [填写] | [填写] | [填写] | + +## Idea 切换日志 + +| 日期 | 从 | 到 | 原因 | +|---|---|---|---| +| [填写] | [填写] | [填写] | [填写] | +| [填写] | [填写] | [填写] | [填写] | + diff --git a/.claude/templates/RESEARCH_BRIEF_CN.md b/.claude/templates/RESEARCH_BRIEF_CN.md new file mode 100644 index 0000000..9eb77c3 --- /dev/null +++ b/.claude/templates/RESEARCH_BRIEF_CN.md @@ -0,0 +1,59 @@ +# 研究简报模板 + +> 用于在研究开始前快速对齐问题、背景、约束和预期方向。 + +## 问题陈述 + +### 1. 问题定义 +[填写：用 2-3 段描述你要解决的核心问题、为什么重要、当前卡点是什么。] + +### 2. 研究动机 +[填写：说明这个问题为什么值得做，现有方法在哪些方面不能满足目标。] + +### 3. 预期影响 +[填写：如果研究成功，可能带来的理论价值、工程价值或论文价值。] + +## 背景 + +### 领域 +[填写：所属大领域，例如 NLP / CV / RL / 系统 / 理论。] + +### 子方向 +[填写：更具体的子方向，例如 长文本推理 / 多模态检索 / 安全对齐。] + +### 已读关键论文 +[填写：列出已经读过的关键论文、年份、核心结论和与你的问题的关系。] + +### 已尝试的方法 +[填写：列出你已经试过的方法、实现方式、实验设置和结果。] + +### 失败经验 +[填写：明确记录失败尝试、失败现象、可能原因和你排除过的解释。] + +## 约束条件 + +| 约束项 | 说明 | +|---|---| +| 算力 | [填写：GPU 型号、数量、可用时长、预算限制] | +| 时间线 | [填写：里程碑日期、截止时间、可用工期] | +| 目标会议 | [填写：目标会议 / 期刊 / 内部评审节点] | + +## 期望方向 + +- [ ] 从零探索 +- [ ] 改进现有方法 +- [ ] 诊断型研究 +- [ ] 其他：[填写] + +## 领域知识 + +[填写：记录你已经确认的领域事实、经验法则、常见坑、重要术语和默认设定。] + +## 非目标 + +[填写：明确哪些问题这次不做，避免范围漂移。] + +## 已有结果 + +[填写：总结目前最重要的结果、指标、可视化、失败/成功信号。] + diff --git a/.claude/templates/RESEARCH_CONTRACT_CN.md b/.claude/templates/RESEARCH_CONTRACT_CN.md new file mode 100644 index 0000000..3b80e24 --- /dev/null +++ b/.claude/templates/RESEARCH_CONTRACT_CN.md @@ -0,0 +1,66 @@ +# 研究契约模板 + +> 用于把研究目标、方法、证据标准和状态同步到一个稳定文档中。 + +## 选定 Idea + +### 描述 +[填写：选中的 idea 是什么，它解决什么问题。] + +### 来源 +[填写：来自哪次讨论、哪篇论文、哪个 review、哪个实验分支。] + +### 选择理由 +[填写：为什么最终选它，而不是其他候选。] + +## 核心 Claims + +| Claim 编号 | 核心主张 | 预期证据 | 当前状态 | +|---|---|---|---| +| C1 | [填写] | [填写] | [填写] | +| C2 | [填写] | [填写] | [填写] | +| C3 | [填写] | [填写] | [填写] | + +## 方法摘要 + +[填写：用简洁语言描述方法框架、关键模块、训练/推理流程和与基线的差异。] + +## 实验设计 + +| 项目 | 说明 | +|---|---| +| 数据集 | [填写] | +| 基线 | [填写] | +| 指标 | [填写] | +| 关键超参 | [填写] | +| 算力 | [填写] | + +## 基线表格 + +| method | dataset | metric | score | source | +|---|---|---:|---:|---| +| [填写] | [填写] | [填写] | [填写] | [填写] | +| [填写] | [填写] | [填写] | [填写] | [填写] | + +## 当前结果表格 + +| method | dataset | metric | score | 备注 | +|---|---|---:|---:|---| +| [填写] | [填写] | [填写] | [填写] | [填写] | +| [填写] | [填写] | [填写] | [填写] | [填写] | + +## 关键决策 + +| 日期 | 决策 | 依据 | 影响 | +|---|---|---|---| +| [填写] | [填写] | [填写] | [填写] | +| [填写] | [填写] | [填写] | [填写] | + +## 状态清单 + +- [ ] 关键假设已明确 +- [ ] 基线已复现 +- [ ] 主实验已定义 +- [ ] 失败案例已记录 +- [ ] 结论已能支持核心 claims + diff --git a/.claude/tools/arxiv_fetch.py b/.claude/tools/arxiv_fetch.py new file mode 100644 index 0000000..bfa36d5 --- /dev/null +++ b/.claude/tools/arxiv_fetch.py @@ -0,0 +1,185 @@ +"""arXiv 搜索与 PDF 下载的独立命令行工具。""" + +from __future__ import annotations + +import argparse +import json +import sys +import time +import xml.etree.ElementTree as ET +from pathlib import Path +from typing import Any +from urllib.parse import quote +from urllib.request import Request, urlopen + + +ATOM_NS = "http://www.w3.org/2005/Atom" +ARXIV_API_URL = ( + "http://export.arxiv.org/api/query?search_query=all:{query}&max_results={max}" +) +ARXIV_PDF_URL = "https://arxiv.org/pdf/{arxiv_id}.pdf" +USER_AGENT = "arxiv_fetch.py/1.0" +MIN_PDF_SIZE = 10 * 1024 +DOWNLOAD_DELAY_SECONDS = 1.0 + + +def _clean_text(value: str) -> str: + """压缩空白并清理文本内容。""" + return " ".join(value.split()) + + +def _normalize_arxiv_id(raw_id: str) -> str: + """将 arXiv 标识归一化为不带版本号的论文 ID。""" + value = raw_id.strip() + if value.startswith("http://") or value.startswith("https://"): + value = value.rsplit("/", 1)[-1] + if value.startswith("abs/") or value.startswith("pdf/"): + value = value.split("/", 1)[1] + if value.endswith(".pdf"): + value = value[:-4] + if value.startswith("arXiv:") or value.startswith("arxiv:"): + value = value.split(":", 1)[1] + if "v" in value: + head, tail = value.rsplit("v", 1) + if tail.isdigit(): + value = head + return value + + +def _request_text(url: str) -> bytes: + """发起 HTTP 请求并返回原始字节内容。""" + request = Request(url, headers={"User-Agent": USER_AGENT}) + with urlopen(request) as response: + return response.read() + + +def _parse_arxiv_feed(xml_bytes: bytes) -> list[dict[str, Any]]: + """解析 arXiv Atom Feed 并提取论文条目。""" + root = ET.fromstring(xml_bytes) + namespace = {"atom": ATOM_NS} + results: list[dict[str, Any]] = [] + + for entry in root.findall("atom:entry", namespace): + title = _clean_text( + entry.findtext("atom:title", default="", namespaces=namespace) + ) + abstract = _clean_text( + entry.findtext("atom:summary", default="", namespaces=namespace) + ) + published = _clean_text( + entry.findtext("atom:published", default="", namespaces=namespace) + ) + if "T" in published: + published = published.split("T", 1)[0] + + authors: list[str] = [] + for author in entry.findall("atom:author", namespace): + name = _clean_text( + author.findtext("atom:name", default="", namespaces=namespace) + ) + if name: + authors.append(name) + + categories: list[str] = [] + for category in entry.findall("atom:category", namespace): + term = category.get("term", "").strip() + if term and term not in categories: + categories.append(term) + + raw_id = entry.findtext("atom:id", default="", namespaces=namespace) + arxiv_id = _normalize_arxiv_id(raw_id.rsplit("/", 1)[-1] if raw_id else "") + + results.append( + { + "title": title, + "authors": authors, + "arxiv_id": arxiv_id, + "abstract": abstract, + "categories": categories, + "published": published, + } + ) + + return results + + +def search_arxiv(query: str, max_results: int) -> list[dict[str, Any]]: + """搜索 arXiv 并返回结构化论文结果。""" + encoded_query = quote(query, safe="") + url = ARXIV_API_URL.format(query=encoded_query, max=max_results) + feed_bytes = _request_text(url) + return _parse_arxiv_feed(feed_bytes) + + +def download_arxiv_pdf(arxiv_id: str, target_dir: Path) -> Path: + """下载指定 arXiv 论文的 PDF 并校验文件大小。""" + normalized_id = _normalize_arxiv_id(arxiv_id) + target_dir.mkdir(parents=True, exist_ok=True) + output_path = target_dir / f"{normalized_id}.pdf" + pdf_url = ARXIV_PDF_URL.format(arxiv_id=normalized_id) + + request = Request(pdf_url, headers={"User-Agent": USER_AGENT}) + time.sleep(DOWNLOAD_DELAY_SECONDS) + try: + with urlopen(request) as response, output_path.open("wb") as handle: + while True: + chunk = response.read(8192) + if not chunk: + break + handle.write(chunk) + except Exception: + if output_path.exists(): + output_path.unlink() + raise + + if output_path.stat().st_size <= MIN_PDF_SIZE: + output_path.unlink(missing_ok=True) + raise ValueError(f"下载文件过小，疑似失败: {output_path}") + + return output_path + + +def build_parser() -> argparse.ArgumentParser: + """构建命令行参数解析器。""" + parser = argparse.ArgumentParser(description="arXiv 搜索与 PDF 下载工具") + subparsers = parser.add_subparsers(dest="command", required=True) + + search_parser = subparsers.add_parser("search", help="搜索 arXiv 论文") + search_parser.add_argument("query", help="搜索关键词") + search_parser.add_argument("--max", type=int, default=10, help="最大返回数量") + + download_parser = subparsers.add_parser("download", help="下载 arXiv PDF") + download_parser.add_argument("arxiv_id", help="arXiv 论文 ID") + download_parser.add_argument( + "--dir", dest="target_dir", default=".", help="保存目录" + ) + + return parser + + +def main(argv: list[str] | None = None) -> int: + """CLI 入口。""" + parser = build_parser() + args = parser.parse_args(argv) + + try: + if args.command == "search": + results = search_arxiv(args.query, args.max) + json.dump(results, sys.stdout, ensure_ascii=False, indent=2) + sys.stdout.write("\n") + return 0 + + if args.command == "download": + output_path = download_arxiv_pdf(args.arxiv_id, Path(args.target_dir)) + sys.stdout.write(f"{output_path}\n") + return 0 + except Exception as exc: + print(f"错误: {exc}", file=sys.stderr) + return 1 + + parser.error(f"未知命令: {args.command}") + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.claude/tools/deepxiv_fetch.py b/.claude/tools/deepxiv_fetch.py new file mode 100644 index 0000000..deda6a7 --- /dev/null +++ b/.claude/tools/deepxiv_fetch.py @@ -0,0 +1,52 @@ +"""DeepXiv CLI 的轻量适配器。""" + +from __future__ import annotations + +import argparse +import shutil +import subprocess +import sys +from typing import cast + +if shutil.which("deepxiv") is None: + print("警告: 未找到 deepxiv CLI，已跳过 DeepXiv 适配器。", file=sys.stderr) + sys.exit(0) + + +def _run_deepxiv(arguments: list[str]) -> int: + """调用底层 deepxiv 命令并透传输出。""" + completed = subprocess.run(["deepxiv", *arguments], check=False) + return completed.returncode + + +def build_parser() -> argparse.ArgumentParser: + """构建命令行参数解析器。""" + parser = argparse.ArgumentParser(description="DeepXiv 命令适配器") + subparsers = parser.add_subparsers(dest="command", required=True) + + for command_name in ("search", "paper-brief", "paper-head", "paper-section"): + subparser = subparsers.add_parser( + command_name, help=f"转发 deepxiv {command_name} 命令" + ) + subparser.add_argument( + "args", nargs=argparse.REMAINDER, help="透传给 deepxiv 的参数" + ) + + return parser + + +def main(argv: list[str] | None = None) -> int: + """CLI 入口。""" + parser = build_parser() + args = parser.parse_args(argv) + + try: + command_args = [args.command, *cast(list[str], args.args)] + return _run_deepxiv(command_args) + except OSError as exc: + print(f"错误: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.claude/tools/exa_search.py b/.claude/tools/exa_search.py new file mode 100644 index 0000000..1879942 --- /dev/null +++ b/.claude/tools/exa_search.py @@ -0,0 +1,144 @@ +"""Exa 搜索命令行工具。""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from dataclasses import asdict, is_dataclass +from typing import Any + +try: + import exa +except ImportError: + print("警告: 未安装 exa-py SDK，已跳过 Exa CLI。", file=sys.stderr) + sys.exit(0) + + +def _to_jsonable(value: Any) -> Any: + """将任意 SDK 返回值递归转换为可序列化对象。""" + if value is None or isinstance(value, (str, int, float, bool)): + return value + if isinstance(value, bytes): + return value.decode("utf-8", errors="replace") + if isinstance(value, dict): + return {str(key): _to_jsonable(item) for key, item in value.items()} + if isinstance(value, (list, tuple, set)): + return [_to_jsonable(item) for item in value] + if is_dataclass(value): + return _to_jsonable(asdict(value)) + model_dump = getattr(value, "model_dump", None) + if callable(model_dump): + return _to_jsonable(model_dump()) + if hasattr(value, "__dict__"): + data = { + key: _to_jsonable(item) + for key, item in vars(value).items() + if not key.startswith("_") + } + if data: + return data + return str(value) + + +def _build_contents_option(content_mode: str) -> dict[str, Any]: + """根据命令行参数构造 Exa contents 配置。""" + if content_mode == "text": + return {"text": True} + return {"highlights": True} + + +def _extract_results(response: Any) -> list[dict[str, Any]]: + """从 Exa 响应中提取结果列表并标准化为 JSON 对象。""" + raw_results: Any + if isinstance(response, dict): + raw_results = response.get("results", []) + else: + raw_results = getattr(response, "results", []) + + if not isinstance(raw_results, list): + return [] + + normalized_results: list[dict[str, Any]] = [] + for item in raw_results: + normalized_item = _to_jsonable(item) + if isinstance(normalized_item, dict): + normalized_results.append(normalized_item) + else: + normalized_results.append({"value": normalized_item}) + return normalized_results + + +def search_exa( + query: str, + max_results: int, + category: str | None, + content_mode: str, +) -> list[dict[str, Any]]: + """执行 Exa 搜索并返回结果列表。""" + api_key = os.environ.get("EXA_API_KEY", "").strip() + if not api_key: + print("警告: 缺少 EXA_API_KEY 环境变量，已跳过 Exa CLI。", file=sys.stderr) + sys.exit(0) + + client = exa.Exa(api_key=api_key) + search_kwargs: dict[str, Any] = { + "num_results": max_results, + "contents": _build_contents_option(content_mode), + } + if category: + search_kwargs["category"] = category + + response = client.search(query, **search_kwargs) + return _extract_results(response) + + +def build_parser() -> argparse.ArgumentParser: + """构建命令行参数解析器。""" + parser = argparse.ArgumentParser(description="Exa 搜索工具") + subparsers = parser.add_subparsers(dest="command", required=True) + + search_parser = subparsers.add_parser("search", help="搜索 Exa 索引") + search_parser.add_argument("query", help="搜索关键词") + search_parser.add_argument("--max", dest="max_results", type=int, default=10) + search_parser.add_argument( + "--category", + default=None, + help="数据类别，例如 research paper、news、pdf 等", + ) + search_parser.add_argument( + "--content", + choices=("highlights", "text"), + default="highlights", + help="返回的内容类型", + ) + return parser + + +def main(argv: list[str] | None = None) -> int: + """CLI 入口。""" + parser = build_parser() + args = parser.parse_args(argv) + + try: + if args.command == "search": + results = search_exa( + query=args.query, + max_results=args.max_results, + category=args.category, + content_mode=args.content, + ) + json.dump(results, sys.stdout, ensure_ascii=False, indent=2) + sys.stdout.write("\n") + return 0 + except Exception as exc: + print(f"错误: {exc}", file=sys.stderr) + return 1 + + parser.error(f"未知命令: {args.command}") + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.claude/tools/openalex_fetch.py b/.claude/tools/openalex_fetch.py new file mode 100644 index 0000000..c0a6b0e --- /dev/null +++ b/.claude/tools/openalex_fetch.py @@ -0,0 +1,190 @@ +"""OpenAlex 搜索命令行工具。""" + +from __future__ import annotations + +import argparse +import json +import sys +from typing import Any + +try: + import requests +except ImportError: + print("警告: 未安装 requests，已跳过 OpenAlex CLI。", file=sys.stderr) + sys.exit(0) + + +API_URL = "https://api.openalex.org/works" +REQUEST_TIMEOUT_SECONDS = 30.0 +USER_AGENT = "openalex_fetch.py/1.0" + + +def _clean_text(value: Any) -> str: + """压缩空白并清理文本值。""" + return " ".join(str(value).split()) + + +def _normalize_year_filter(raw_year: str) -> str: + """将命令行年份参数转换为 OpenAlex filter 语法。""" + value = raw_year.strip() + if not value: + raise ValueError("年份参数不能为空") + + if value.isdigit() and len(value) == 4: + return f"publication_year:{value}" + + if value.endswith("-") and value[:-1].strip().isdigit(): + start_year = value[:-1].strip() + return f"from_publication_date:{start_year}-01-01" + + if value.startswith("-") and value[1:].strip().isdigit(): + end_year = value[1:].strip() + return f"to_publication_date:{end_year}-12-31" + + if "-" in value: + start_year, end_year = [part.strip() for part in value.split("-", 1)] + if start_year.isdigit() and end_year.isdigit(): + return f"publication_year:{start_year}-{end_year}" + + raise ValueError(f"无法识别的年份参数: {raw_year}") + + +def _normalize_sort_value(sort_value: str) -> str: + """将用户输入的排序参数转换为 OpenAlex sort 语法。""" + value = sort_value.strip() + if not value: + raise ValueError("排序参数不能为空") + if value in {"relevance", "relevance_score"}: + return "relevance_score:desc" + return value + + +def _normalize_work(work: dict[str, Any]) -> dict[str, Any]: + """将 OpenAlex work 记录压缩成稳定输出结构。""" + authorships = work.get("authorships") + authors: list[str] = [] + institutions: list[str] = [] + if isinstance(authorships, list): + for authorship in authorships: + if not isinstance(authorship, dict): + continue + author = authorship.get("author") + if isinstance(author, dict): + display_name = author.get("display_name") + if display_name: + cleaned_name = _clean_text(display_name) + if cleaned_name not in authors: + authors.append(cleaned_name) + author_institutions = authorship.get("institutions") + if isinstance(author_institutions, list): + for institution in author_institutions: + if not isinstance(institution, dict): + continue + display_name = institution.get("display_name") + if display_name: + cleaned_name = _clean_text(display_name) + if cleaned_name not in institutions: + institutions.append(cleaned_name) + + return { + "title": _clean_text(work.get("display_name", "")), + "authors": authors, + "year": work.get("publication_year"), + "cited_by_count": work.get("cited_by_count"), + "institutions": institutions, + "doi": work.get("doi"), + } + + +def search_openalex( + query: str, + max_results: int, + year: str | None, + work_type: str | None, + sort: str, +) -> list[dict[str, Any]]: + """搜索 OpenAlex 并返回标准化论文结果。""" + params: dict[str, Any] = { + "search": query, + "per-page": max_results, + "sort": _normalize_sort_value(sort), + } + + filters: list[str] = [] + if year: + filters.append(_normalize_year_filter(year)) + if work_type: + filters.append(f"type:{work_type.strip()}") + if filters: + params["filter"] = ",".join(filters) + + response = requests.get( + API_URL, + params=params, + headers={"User-Agent": USER_AGENT}, + timeout=REQUEST_TIMEOUT_SECONDS, + ) + response.raise_for_status() + + payload = response.json() + if not isinstance(payload, dict): + raise ValueError("OpenAlex 返回了非对象类型的 JSON") + + works = payload.get("results", []) + if not isinstance(works, list): + raise ValueError("OpenAlex 响应缺少 results 列表") + + normalized_results: list[dict[str, Any]] = [] + for work in works: + if isinstance(work, dict): + normalized_results.append(_normalize_work(work)) + return normalized_results + + +def build_parser() -> argparse.ArgumentParser: + """构建命令行参数解析器。""" + parser = argparse.ArgumentParser(description="OpenAlex 搜索工具") + subparsers = parser.add_subparsers(dest="command", required=True) + + search_parser = subparsers.add_parser("search", help="搜索 OpenAlex works") + search_parser.add_argument("query", help="搜索关键词") + search_parser.add_argument("--max", dest="max_results", type=int, default=10) + search_parser.add_argument("--year", default=None, help="年份或年份范围") + search_parser.add_argument( + "--type", dest="work_type", default=None, help="作品类型" + ) + search_parser.add_argument( + "--sort", + default="relevance", + help="排序字段，relevance 会映射为 relevance_score:desc", + ) + return parser + + +def main(argv: list[str] | None = None) -> int: + """CLI 入口。""" + parser = build_parser() + args = parser.parse_args(argv) + + try: + if args.command == "search": + results = search_openalex( + query=args.query, + max_results=args.max_results, + year=args.year, + work_type=args.work_type, + sort=args.sort, + ) + json.dump(results, sys.stdout, ensure_ascii=False, indent=2) + sys.stdout.write("\n") + return 0 + except (requests.RequestException, ValueError) as exc: + print(f"错误: {exc}", file=sys.stderr) + return 1 + + parser.error(f"未知命令: {args.command}") + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.claude/tools/research_wiki.py b/.claude/tools/research_wiki.py new file mode 100644 index 0000000..f56900e --- /dev/null +++ b/.claude/tools/research_wiki.py @@ -0,0 +1,681 @@ +"""研究 Wiki 的初始化与日志工具。""" + +from __future__ import annotations + +import argparse +import json +import re +import unicodedata +from datetime import datetime, timezone +from pathlib import Path +from typing import Any +from urllib.request import urlopen +import xml.etree.ElementTree as ET + +ENTITY_TYPES = { + "paper": "papers", + "idea": "ideas", + "experiment": "experiments", + "claim": "claims", + "gap": "gaps", + "design": "designs", + "finding": "findings", + "adr": "adrs", + "plan": "plans", + "review": "reviews", + "schema": "schemas", + "metric": "metrics", +} + +EDGE_TYPES = [ + "extends", + "contradicts", + "supersedes", + "addresses_gap", + "inspired_by", + "tested_by", + "supports", + "invalidates", + "implements", + "reveals", + "informs", + "refines", + "depends_on", + "measures", + "evaluates", +] + +QUERY_PACK_BUDGET = 8000 + + +def slugify(text: str) -> str: + """将文本转换为 URL 安全的 slug。""" + normalized = unicodedata.normalize("NFKD", text) + normalized = re.sub(r"[^\w\s-]", "", normalized.lower()) + return re.sub(r"[-\s]+", "_", normalized).strip("_")[:60] + + +def _write_if_missing(path: Path, content: str) -> None: + """仅在文件不存在时写入内容。 + + 参数: + path: 目标文件路径。 + content: 需要写入的文本内容。 + """ + if path.exists(): + return + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + + +def append_log(wiki_dir: str, message: str) -> None: + """向 Wiki 变更日志追加一条记录。 + + 参数: + wiki_dir: Wiki 根目录。 + message: 需要记录的消息。 + """ + log_path = Path(wiki_dir) / "log.md" + timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC") + with log_path.open("a", encoding="utf-8") as handle: + handle.write(f"- [{timestamp}] {message}\n") + + +def _render_frontmatter(data: dict[str, Any]) -> str: + """将字典渲染为 YAML frontmatter。""" + lines = ["---"] + for key, value in data.items(): + if isinstance(value, list): + rendered_value = json.dumps(value, ensure_ascii=False) + elif isinstance(value, str) and " " in value: + rendered_value = f'"{value}"' + else: + rendered_value = str(value) + lines.append(f"{key}: {rendered_value}") + lines.append("---") + return "\n".join(lines) + "\n" + + +def _read_frontmatter(filepath: Path) -> dict[str, Any]: + """从 markdown 文件读取简易 frontmatter。""" + text = filepath.read_text(encoding="utf-8") + if not text.startswith("---\n"): + return {} + + lines = text.splitlines() + result: dict[str, Any] = {} + for line in lines[1:]: + if line == "---": + break + if ":" not in line: + continue + key, _, raw_value = line.partition(":") + value = raw_value.strip() + if value.startswith("[") and value.endswith("]"): + try: + result[key.strip()] = json.loads(value) + continue + except json.JSONDecodeError: + pass + if len(value) >= 2 and ( + (value.startswith('"') and value.endswith('"')) + or (value.startswith("'") and value.endswith("'")) + ): + value = value[1:-1] + result[key.strip()] = value + return result + + +def _add_node_to_graph( + wiki_dir: str, node_id: str, label: str, entity_type: str +) -> None: + """向关系图中添加节点，若已存在则跳过。""" + graph_path = Path(wiki_dir) / "graph" / "edges.json" + graph = json.loads(graph_path.read_text(encoding="utf-8")) + nodes = graph.setdefault("nodes", []) + if not any(node.get("id") == node_id for node in nodes): + nodes.append({"id": node_id, "label": label, "type": entity_type}) + graph_path.write_text( + json.dumps(graph, ensure_ascii=False, indent=2), encoding="utf-8" + ) + + +def add_edge( + wiki_dir: str, + from_id: str, + to_id: str, + edge_type: str, + evidence: str = "", +) -> None: + """向关系图中添加一条边。""" + if edge_type not in EDGE_TYPES: + raise ValueError(f"未知边类型: {edge_type}，合法值: {EDGE_TYPES}") + + graph_path = Path(wiki_dir) / "graph" / "edges.json" + graph = json.loads(graph_path.read_text(encoding="utf-8")) + links = graph.setdefault("links", []) + + for link in links: + if ( + link.get("source") == from_id + and link.get("target") == to_id + and link.get("relation") == edge_type + ): + return + + links.append( + { + "source": from_id, + "target": to_id, + "relation": edge_type, + "evidence": evidence, + "added": datetime.now(timezone.utc).isoformat(), + } + ) + graph_path.write_text( + json.dumps(graph, ensure_ascii=False, indent=2), encoding="utf-8" + ) + append_log(wiki_dir, f"新增边: {from_id} --{edge_type}--> {to_id}") + + +def add_entity( + wiki_dir: str, + entity_type: str, + entity_id: str, + title: str, + extra_frontmatter: dict[str, Any] | None = None, +) -> Path: + """创建一个实体页面并同步到关系图。""" + if entity_type not in ENTITY_TYPES: + raise ValueError(f"未知实体类型: {entity_type}，合法值: {list(ENTITY_TYPES)}") + + root = Path(wiki_dir) + entity_subdir = root / ENTITY_TYPES[entity_type] + node_id = f"{entity_type}:{entity_id}" + + for existing_path in entity_subdir.glob("*.md"): + frontmatter = _read_frontmatter(existing_path) + if frontmatter.get("node_id") == node_id: + return existing_path + + filepath = entity_subdir / f"{entity_id}.md" + frontmatter: dict[str, Any] = { + "type": entity_type, + "node_id": node_id, + "title": title, + "date": datetime.now(timezone.utc).strftime("%Y-%m-%d"), + } + if extra_frontmatter: + frontmatter.update(extra_frontmatter) + + content = _render_frontmatter(frontmatter) + f"\n# {title}\n\n" + filepath.write_text(content, encoding="utf-8") + + _add_node_to_graph(wiki_dir, node_id, title, entity_type) + append_log(wiki_dir, f"新增 {entity_type}: {title} ({node_id})") + return filepath + + +def _fetch_arxiv_metadata(arxiv_id: str) -> dict[str, Any] | None: + """通过 arXiv Atom API 拉取论文元数据。""" + clean_id = ( + arxiv_id.removeprefix("arxiv:") + .removeprefix("arXiv:") + .removeprefix("ARXIV:") + .strip() + ) + url = f"http://export.arxiv.org/api/query?id_list={clean_id}" + + try: + with urlopen(url) as response: + root = ET.fromstring(response.read()) + except Exception: + return None + + namespace = {"atom": "http://www.w3.org/2005/Atom"} + entry = root.find("atom:entry", namespace) + if entry is None: + return None + + def _text(path: str) -> str: + value = entry.findtext(path, default="", namespaces=namespace) + return " ".join(value.split()) + + authors = [ + " ".join(author.findtext("atom:name", default="", namespaces=namespace).split()) + for author in entry.findall("atom:author", namespace) + ] + authors = [author for author in authors if author] + published = _text("atom:published") + year = published[:4] if len(published) >= 4 else "" + + return { + "title": _text("atom:title"), + "authors": authors, + "year": year, + "abstract": _text("atom:summary"), + "arxiv_id": clean_id, + } + + +def _last_name(full_name: str) -> str: + """提取人名最后一个词作为姓氏。""" + parts = full_name.strip().split() + return parts[-1].lower() if parts else "unknown" + + +def _markdown_body(text: str) -> str: + """提取 markdown 中 frontmatter 之后的正文。""" + if not text.startswith("---\n"): + return text + + lines = text.splitlines() + for index, line in enumerate(lines[1:], start=1): + if line == "---": + return "\n".join(lines[index + 1 :]) + return "" + + +def _iter_entity_pages(wiki_dir: str) -> list[tuple[str, Path, dict[str, Any], str]]: + """收集 Wiki 中所有实体页面。""" + root = Path(wiki_dir) + pages: list[tuple[str, Path, dict[str, Any], str]] = [] + + for entity_type, subdir_name in ENTITY_TYPES.items(): + subdir = root / subdir_name + for path in sorted(subdir.glob("*.md")): + frontmatter = _read_frontmatter(path) + body = _markdown_body(path.read_text(encoding="utf-8")) + resolved_type = ( + str(frontmatter.get("type", entity_type)).strip() or entity_type + ) + pages.append((resolved_type, path, frontmatter, body)) + + return pages + + +def rebuild_index(wiki_dir: str) -> None: + """按实体类型重建索引页。""" + root = Path(wiki_dir) + timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC") + pages = _iter_entity_pages(wiki_dir) + + grouped: dict[str, list[tuple[Path, dict[str, Any]]]] = {} + for entity_type, path, frontmatter, _ in pages: + grouped.setdefault(entity_type, []).append((path, frontmatter)) + + lines = [ + "# Research Wiki 索引", + "", + f"> 自动生成，更新时间：{timestamp}", + "", + ] + + for entity_type in ENTITY_TYPES: + entries = grouped.get(entity_type, []) + if not entries: + continue + entries = sorted( + entries, + key=lambda item: ( + str(item[1].get("title", item[0].stem)).casefold(), + item[0].name, + ), + ) + lines.append(f"## {entity_type} ({len(entries)})") + for path, frontmatter in entries: + title = str(frontmatter.get("title", path.stem)).strip() or path.stem + node_id = str( + frontmatter.get("node_id", f"{entity_type}:{path.stem}") + ).strip() + relative_path = path.relative_to(root).as_posix() + lines.append(f"- [{title}]({relative_path}) `{node_id}`") + lines.append("") + + content = "\n".join(lines).rstrip() + "\n" + (root / "index.md").write_text(content, encoding="utf-8") + append_log(wiki_dir, f"重建索引: {len(pages)} 篇页面") + + +def rebuild_query_pack(wiki_dir: str) -> None: + """基于 frontmatter 重新生成 Query Pack。""" + pages = _iter_entity_pages(wiki_dir) + + failed_ideas: list[tuple[str, str]] = [] + open_gaps: list[str] = [] + core_papers: list[tuple[str, str, str]] = [] + + for entity_type, path, frontmatter, _ in pages: + title = str(frontmatter.get("title", path.stem)).strip() or path.stem + if entity_type == "idea": + outcome = str(frontmatter.get("outcome", "")).strip().lower() + if outcome in {"negative", "failed"}: + failure_reason = str(frontmatter.get("failure_reason", "")).strip() + failed_ideas.append((title, failure_reason or "未填写")) + if entity_type == "gap": + status = str(frontmatter.get("status", "open")).strip().lower() + if not status or status == "open": + open_gaps.append(title) + if entity_type == "paper": + year = str(frontmatter.get("year", "")).strip() + venue = str(frontmatter.get("venue", "")).strip() + core_papers.append( + ( + title, + year or "未知年份", + venue or "未知 venue", + ) + ) + + failed_ideas.sort(key=lambda item: item[0].casefold()) + open_gaps.sort(key=str.casefold) + core_papers.sort(key=lambda item: (item[1], item[0].casefold(), item[2].casefold())) + + sections: list[tuple[str, list[str]]] = [] + if failed_ideas: + sections.append( + ( + "## 失败 Idea（禁区）", + [f"- ❌ {title}: {reason}" for title, reason in failed_ideas[:10]], + ) + ) + if open_gaps: + sections.append( + ( + "## 未解决空白", + [f"- {title}" for title in open_gaps[:5]], + ) + ) + if core_papers: + sections.append( + ( + "## 核心论文", + [ + f"- [{title}] ({year}) {venue}" + for title, year, venue in core_papers[:12] + ], + ) + ) + + content_parts = [ + "# Query Pack", + "", + "> 自动生成，请勿手动编辑。", + "", + "", + ] + current_content = "\n".join(content_parts) + + for section_title, section_lines in sections: + section_text = "\n".join([section_title, *section_lines, "", ""]) + candidate = current_content + section_text + if len(candidate) > QUERY_PACK_BUDGET: + break + current_content = candidate + + content = current_content.rstrip() + "\n" + (Path(wiki_dir) / "query_pack.md").write_text(content, encoding="utf-8") + append_log(wiki_dir, f"重建 Query Pack: {len(content)} 字符") + + +def get_stats(wiki_dir: str) -> str: + """统计 Wiki 规模与图谱信息。""" + root = Path(wiki_dir) + lines: list[str] = [] + + for entity_type, subdir_name in ENTITY_TYPES.items(): + count = len(list((root / subdir_name).glob("*.md"))) + lines.append(f"{entity_type}: {count}") + + graph_path = root / "graph" / "edges.json" + if graph_path.exists(): + graph = json.loads(graph_path.read_text(encoding="utf-8")) + else: + graph = {"nodes": [], "links": []} + + lines.append(f"nodes: {len(graph.get('nodes', []))}") + lines.append(f"links: {len(graph.get('links', []))}") + return "\n".join(lines) + + +def lint_wiki(wiki_dir: str) -> list[str]: + """检查 Wiki 中的孤立节点与稀疏页面。""" + root = Path(wiki_dir) + issues: list[str] = [] + + graph_path = root / "graph" / "edges.json" + if graph_path.exists(): + graph = json.loads(graph_path.read_text(encoding="utf-8")) + else: + graph = {"nodes": [], "links": []} + + linked_node_ids = { + str(link.get("source", "")).strip() + for link in graph.get("links", []) + if str(link.get("source", "")).strip() + } + linked_node_ids.update( + str(link.get("target", "")).strip() + for link in graph.get("links", []) + if str(link.get("target", "")).strip() + ) + + for node in graph.get("nodes", []): + node_id = str(node.get("id", "")).strip() + if node_id and node_id not in linked_node_ids: + label = str(node.get("label", "")).strip() + issues.append(f"孤立节点: {node_id}" + (f" ({label})" if label else "")) + + for _, path, _, body in _iter_entity_pages(wiki_dir): + if len(body.strip()) < 50: + relative_path = path.relative_to(root).as_posix() + issues.append(f"稀疏页面: {relative_path} (正文 {len(body.strip())} 字符)") + + return issues + + +def ingest_paper( + wiki_dir: str, + arxiv_id: str | None = None, + title: str | None = None, + authors: str | None = None, + year: str | None = None, + venue: str = "arXiv", + thesis: str = "", +) -> Path | None: + """创建 paper 实体并补充标准化正文模板。""" + abstract = "" + authors_list: list[str] + paper_title = title + paper_year = year + paper_arxiv_id = None + + if arxiv_id: + metadata = _fetch_arxiv_metadata(arxiv_id) + if metadata is None: + return None + paper_title = metadata["title"] + authors_list = metadata["authors"] + paper_year = metadata["year"] + abstract = metadata["abstract"] + paper_arxiv_id = metadata["arxiv_id"] + else: + if not (title and authors and year): + raise ValueError("非 arXiv 论文需提供 --title、--authors、--year") + authors_list = [ + author.strip() for author in authors.split(",") if author.strip() + ] + if not authors_list: + raise ValueError("authors 不能为空") + + if not paper_title: + raise ValueError("论文标题不能为空") + if not paper_year: + raise ValueError("论文年份不能为空") + if not authors_list: + raise ValueError("论文作者不能为空") + + slug = f"{_last_name(authors_list[0])}{paper_year}_{slugify(paper_title)[:30]}" + extra_frontmatter: dict[str, Any] = { + "authors": authors_list, + "year": paper_year, + "venue": venue, + } + if paper_arxiv_id is not None: + extra_frontmatter["arxiv_id"] = paper_arxiv_id + + path = add_entity(wiki_dir, "paper", slug, paper_title, extra_frontmatter) + + content = path.read_text(encoding="utf-8") + if "## 一句话论点" not in content: + body = f"\n## 一句话论点\n\n{thesis or '待填写'}\n\n" + if abstract: + body += f"## 摘要\n\n> {' '.join(abstract.split())}\n\n" + body += ( + "## 问题/空白\n\n" + "## 方法\n\n" + "## 关键结果\n\n" + "## 局限性\n\n" + "## 与本项目的相关性\n\n" + ) + path.write_text(content + body, encoding="utf-8") + + return path + + +def init_wiki(wiki_dir: str) -> None: + """初始化研究 Wiki 的目录结构与基础文件。 + + 参数: + wiki_dir: Wiki 根目录。 + """ + root = Path(wiki_dir) + root.mkdir(parents=True, exist_ok=True) + + for entity_dir in ENTITY_TYPES.values(): + (root / entity_dir).mkdir(parents=True, exist_ok=True) + + graph_dir = root / "graph" + graph_dir.mkdir(parents=True, exist_ok=True) + + _write_if_missing( + root / "index.md", "# Research Wiki 索引\n\n> 自动生成，请勿手动编辑。\n" + ) + _write_if_missing(root / "log.md", "# Wiki 变更日志\n\n") + _write_if_missing(root / "gap_map.md", "# 领域空白汇总\n\n") + _write_if_missing( + root / "query_pack.md", + "# Query Pack\n\n> 尚无数据。运行 research-lit 或 idea-creator 后自动生成。\n", + ) + _write_if_missing( + graph_dir / "edges.json", + json.dumps({"directed": True, "nodes": [], "links": []}, ensure_ascii=False), + ) + + append_log(wiki_dir, "Wiki 初始化完成") + + +def main() -> None: + """命令行入口。""" + parser = argparse.ArgumentParser(description="研究 Wiki 工具") + subparsers = parser.add_subparsers(dest="command", required=True) + + init_parser = subparsers.add_parser("init", help="初始化 Wiki") + init_parser.add_argument("wiki_dir", help="Wiki 根目录") + + log_parser = subparsers.add_parser("log", help="追加日志") + log_parser.add_argument("wiki_dir", help="Wiki 根目录") + log_parser.add_argument("message", nargs="+", help="日志内容") + + entity_parser = subparsers.add_parser("add_entity", help="创建实体") + entity_parser.add_argument("wiki_dir", help="Wiki 根目录") + entity_parser.add_argument( + "--type", required=True, choices=list(ENTITY_TYPES), help="实体类型" + ) + entity_parser.add_argument("--id", required=True, help="实体 ID") + entity_parser.add_argument("--title", required=True, help="实体标题") + + edge_parser = subparsers.add_parser("add_edge", help="创建关系边") + edge_parser.add_argument("wiki_dir", help="Wiki 根目录") + edge_parser.add_argument( + "--from", dest="from_id", required=True, help="起点节点 ID" + ) + edge_parser.add_argument("--to", dest="to_id", required=True, help="终点节点 ID") + edge_parser.add_argument( + "--type", required=True, choices=EDGE_TYPES, dest="edge_type", help="边类型" + ) + edge_parser.add_argument("--evidence", default="", help="证据说明") + + ingest_parser = subparsers.add_parser("ingest_paper", help="导入论文") + ingest_parser.add_argument("wiki_dir", help="Wiki 根目录") + ingest_parser.add_argument("--arxiv-id", dest="arxiv_id", help="arXiv ID") + ingest_parser.add_argument("--title", help="论文标题") + ingest_parser.add_argument("--authors", help="作者，逗号分隔") + ingest_parser.add_argument("--year", help="年份") + ingest_parser.add_argument("--venue", default="arXiv", help="发表 venue") + ingest_parser.add_argument("--thesis", default="", help="一句话论点") + + rebuild_query_pack_parser = subparsers.add_parser( + "rebuild_query_pack", help="重建 Query Pack" + ) + rebuild_query_pack_parser.add_argument("wiki_dir", help="Wiki 根目录") + + rebuild_index_parser = subparsers.add_parser("rebuild_index", help="重建索引") + rebuild_index_parser.add_argument("wiki_dir", help="Wiki 根目录") + + stats_parser = subparsers.add_parser("stats", help="输出统计信息") + stats_parser.add_argument("wiki_dir", help="Wiki 根目录") + + lint_parser = subparsers.add_parser("lint", help="检查 Wiki 问题") + lint_parser.add_argument("wiki_dir", help="Wiki 根目录") + + args = parser.parse_args() + + if args.command == "init": + init_wiki(args.wiki_dir) + return + + if args.command == "log": + append_log(args.wiki_dir, " ".join(args.message)) + return + + if args.command == "add_entity": + add_entity(args.wiki_dir, args.type, args.id, args.title) + return + + if args.command == "add_edge": + add_edge(args.wiki_dir, args.from_id, args.to_id, args.edge_type, args.evidence) + return + + if args.command == "ingest_paper": + ingest_paper( + args.wiki_dir, + arxiv_id=args.arxiv_id, + title=args.title, + authors=args.authors, + year=args.year, + venue=args.venue, + thesis=args.thesis, + ) + return + + if args.command == "rebuild_query_pack": + rebuild_query_pack(args.wiki_dir) + return + + if args.command == "rebuild_index": + rebuild_index(args.wiki_dir) + return + + if args.command == "stats": + print(get_stats(args.wiki_dir)) + return + + if args.command == "lint": + issues = lint_wiki(args.wiki_dir) + print("\n".join(issues) if issues else "未发现问题") + return + + +if __name__ == "__main__": + main() diff --git a/.claude/tools/semantic_scholar_fetch.py b/.claude/tools/semantic_scholar_fetch.py new file mode 100644 index 0000000..ad72db8 --- /dev/null +++ b/.claude/tools/semantic_scholar_fetch.py @@ -0,0 +1,224 @@ +"""Semantic Scholar 搜索命令行工具。""" + +from __future__ import annotations + +import argparse +import json +import sys +import time +from typing import Any + +try: + import requests +except ImportError: + print("警告: 未安装 requests，已跳过 Semantic Scholar CLI。", file=sys.stderr) + sys.exit(0) + + +API_URL = "https://api.semanticscholar.org/graph/v1/paper/search" +REQUEST_FIELDS = "title,authors,year,venue,citationCount,externalIds,tldr" +REQUEST_INTERVAL_SECONDS = 1.0 +REQUEST_TIMEOUT_SECONDS = 30.0 +USER_AGENT = "semantic_scholar_fetch.py/1.0" +_LAST_REQUEST_AT: float | None = None + + +def _normalize_filter_values(values: list[str] | None) -> str | None: + """将过滤参数整理为 API 需要的逗号分隔字符串。""" + if not values: + return None + + items: list[str] = [] + for value in values: + for item in value.split(","): + cleaned = item.strip() + if cleaned: + items.append(cleaned) + + if not items: + return None + return ",".join(items) + + +def _respect_rate_limit() -> None: + """确保连续请求之间至少间隔一秒。""" + global _LAST_REQUEST_AT + + now = time.monotonic() + if _LAST_REQUEST_AT is not None: + elapsed = now - _LAST_REQUEST_AT + if elapsed < REQUEST_INTERVAL_SECONDS: + time.sleep(REQUEST_INTERVAL_SECONDS - elapsed) + + _LAST_REQUEST_AT = time.monotonic() + + +def _clean_text(value: Any) -> str: + """压缩并清理任意文本值。""" + return " ".join(str(value).split()) + + +def _normalize_authors(authors: Any) -> list[str]: + """将作者字段统一为作者姓名列表。""" + normalized_authors: list[str] = [] + if not isinstance(authors, list): + return normalized_authors + + for author in authors: + if isinstance(author, dict): + name = author.get("name") + if name: + normalized_authors.append(_clean_text(name)) + elif author: + normalized_authors.append(_clean_text(author)) + + return normalized_authors + + +def _normalize_external_ids(external_ids: Any) -> dict[str, str]: + """将 externalIds 统一整理为字符串字典，并保留 arXiv 标识。""" + normalized: dict[str, str] = {} + if isinstance(external_ids, dict): + for key, value in external_ids.items(): + if value is None: + continue + cleaned_value = _clean_text(value) + if cleaned_value: + normalized[str(key)] = cleaned_value + + arxiv_id = ( + normalized.get("ArXiv") or normalized.get("arXiv") or normalized.get("arxiv") + ) + if arxiv_id and "ArXiv" not in normalized: + normalized["ArXiv"] = arxiv_id + + return normalized + + +def _normalize_tldr(tldr: Any) -> Any: + """保留 TLDR 字段原始结构，但去除明显的空字符串。""" + if isinstance(tldr, dict): + normalized_tldr: dict[str, Any] = {} + for key, value in tldr.items(): + if isinstance(value, str): + cleaned = _clean_text(value) + if cleaned: + normalized_tldr[key] = cleaned + elif value is not None: + normalized_tldr[key] = value + return normalized_tldr or None + return tldr + + +def _normalize_paper(paper: dict[str, Any]) -> dict[str, Any]: + """将 API 返回的论文记录压缩成稳定的输出结构。""" + return { + "title": _clean_text(paper.get("title", "")), + "authors": _normalize_authors(paper.get("authors")), + "year": paper.get("year"), + "venue": _clean_text(paper.get("venue", "")), + "citationCount": paper.get("citationCount"), + "externalIds": _normalize_external_ids(paper.get("externalIds")), + "tldr": _normalize_tldr(paper.get("tldr")), + } + + +def _request_json(url: str, params: dict[str, Any]) -> dict[str, Any]: + """发起受限频率控制的 GET 请求并返回 JSON 对象。""" + _respect_rate_limit() + response = requests.get( + url, + params=params, + headers={"User-Agent": USER_AGENT}, + timeout=REQUEST_TIMEOUT_SECONDS, + ) + response.raise_for_status() + + payload = response.json() + if not isinstance(payload, dict): + raise ValueError("Semantic Scholar 返回了非对象类型的 JSON") + return payload + + +def search_papers( + query: str, + max_results: int, + fields_of_study: list[str] | None = None, + publication_types: list[str] | None = None, +) -> list[dict[str, Any]]: + """搜索 Semantic Scholar 论文并返回标准化结果。""" + params: dict[str, Any] = { + "query": query, + "limit": max_results, + "fields": REQUEST_FIELDS, + } + + normalized_fields_of_study = _normalize_filter_values(fields_of_study) + if normalized_fields_of_study is not None: + params["fieldsOfStudy"] = normalized_fields_of_study + + normalized_publication_types = _normalize_filter_values(publication_types) + if normalized_publication_types is not None: + params["publicationTypes"] = normalized_publication_types + + payload = _request_json(API_URL, params) + papers = payload.get("data", []) + if not isinstance(papers, list): + raise ValueError("Semantic Scholar 响应缺少 data 列表") + + normalized_papers: list[dict[str, Any]] = [] + for paper in papers: + if isinstance(paper, dict): + normalized_papers.append(_normalize_paper(paper)) + return normalized_papers + + +def build_parser() -> argparse.ArgumentParser: + """构建命令行参数解析器。""" + parser = argparse.ArgumentParser(description="Semantic Scholar 搜索工具") + subparsers = parser.add_subparsers(dest="command", required=True) + + search_parser = subparsers.add_parser("search", help="搜索论文") + search_parser.add_argument("query", help="搜索关键词") + search_parser.add_argument("--max", dest="max_results", type=int, default=10) + search_parser.add_argument( + "--fields-of-study", + nargs="+", + default=None, + help="按学科领域过滤", + ) + search_parser.add_argument( + "--publication-types", + nargs="+", + default=None, + help="按出版类型过滤", + ) + return parser + + +def main(argv: list[str] | None = None) -> int: + """CLI 入口。""" + parser = build_parser() + args = parser.parse_args(argv) + + try: + if args.command == "search": + results = search_papers( + query=args.query, + max_results=args.max_results, + fields_of_study=args.fields_of_study, + publication_types=args.publication_types, + ) + json.dump(results, sys.stdout, ensure_ascii=False, indent=2) + sys.stdout.write("\n") + return 0 + except (requests.RequestException, ValueError) as exc: + print(f"错误: {exc}", file=sys.stderr) + return 1 + + parser.error(f"未知命令: {args.command}") + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..d3a4291 --- /dev/null +++ b/.env.example @@ -0,0 +1,27 @@ +# PolyGateway 工程配置模板（复制为 .env 使用；.env 不提交） +# 键名清单在 M1 设计文档定稿后补全；以下为已定的韧性参数键名风格 +# （沿用三个参考项目的习惯，降低迁移改名成本，见 ARCHITECTURE.md §9）。 + +# ── 多源配置（{SCOPE}__{PROVIDER}__{N}__{FIELD}）── +# LLM__QWEN__1__BASE_URL= +# LLM__QWEN__1__API_KEY= +# LLM__QWEN__1__MODEL= + +# ── 韧性参数 ── +# LLM_TIMEOUT=120 +# LLM_MAX_RETRIES=3 +# LLM_RETRY_BASE_DELAY=2.0 +# LLM_RETRY_MAX_DELAY=30.0 +# LLM_CIRCUIT_BREAKER_THRESHOLD=48 +# LLM_CIRCUIT_BREAKER_COOLDOWN=60 +# LLM_TTFT_TIMEOUT=30 +# LLM_INTER_TOKEN_TIMEOUT=15 + +# ── 后端选择（命名 M1 定稿）── +# PGW_LIMITER_BACKEND=memory +# PGW_TELEMETRY_BACKEND=sqlite +# PGW_QUOTA_FULL=wait + +# ── Redis（缓存 / 分布式限流熔断）── +# REDIS_URL=redis://localhost:6379/0 +# REDIS_CACHE_TTL=86400 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e258f6a --- /dev/null +++ b/.gitignore @@ -0,0 +1,32 @@ +# 参考项目：只读且不入库（CLAUDE.md 硬性规则） +reference/ + +# 密钥与工程配置（模板 .env.example 入库） +.env +.env.* +!.env.example + +# 运行产物（不提交） +data/ +logs/ +tests/outputs/ +results/ +*.db +*.sqlite3 + +# Python +__pycache__/ +*.py[cod] +*.egg-info/ +build/ +dist/ +.pytest_cache/ +.ruff_cache/ +.coverage +htmlcov/ + +# Claude Code 本地配置（项目级 settings.json 入库） +.claude/settings.local.json + +# 系统 +.DS_Store diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..ea9203c --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,142 @@ +# CLAUDE.md + +> [!URGENT] +> **实验室内部通用基础库(生产级、非 MVP、零业务假设)** +> 1. 本项目是被多个科研/生产项目依赖的**库**,不是应用:稳定性、并发性、防御性、可观测与测试不可为"简单"让步(YAGNI 仍适用,但不削减健壮性)。库的 bug 会同时击穿所有下游项目。 +> 2. 你的所有思考过程和回复必须使用 **简体中文**。 + +## 1. 项目元数据 +- **核心目标**: PolyGateway = 统一的大语言模型(LLM/VLM/OCR,音频预留)调度与中转库。治理单位是**一次模型调用**:请求封装、多源多账号、限流、错误分类与重试、熔断、Redis 响应缓存、流式看门狗、遥测(含成本)、结构化输出策略。全组件端口化可插拔。 +- **架构权威文档**: `research-wiki/ARCHITECTURE.md`(架构单一事实源,含 D1-D12 决策及讨论过程、子系统设计、三项目迁移验收标准;**不受 400 行设计文档限制**,以无歧义传达既有讨论为准绳)。`research-wiki/designs/` 仅存放每次实现具体功能的设计文档。 +- **参考项目**: `reference/` 下三个项目是本库的需求来源与代码蓝本(**只读,勿改**);库必须能按 ARCHITECTURE.md §11 被它们迁移接入,否则即边界缺口。 +- **技术栈**: Python 3.11+,核心仅依赖 `httpx` + `pydantic`,其余(redis/sqlite/postgres/json_repair/openai)一律 optional extras。conda 环境 `PolyGateway`。 + +## 2. 常用命令 + +> [!CRITICAL] +> 所有 Python 命令必须在 `PolyGateway` conda 环境中执行(`conda run -n PolyGateway <cmd>` 或先激活)。长时间运行的程序用 tmux 且禁用日志缓存。 + +```bash +make install # editable 安装(含 dev 与全部 extras) +make test # pytest + 覆盖率 +make lint # ruff --fix + import-linter(依赖铁律机械化执法) +make format # ruff format +make ci # 只读验证(check + test) +``` + +## 3. 标准作业程序(分档触发) + +> **档位原则(Fable 5 适配,2026-07 调研决策)**: 约束"边界与验收",不规定思考步骤。强制档(MANDATORY)是硬门;其余由模型按 skill description 自判,自判标准是任务实质(规模/风险/是否触及公共承诺),不是省事。硬边界(reference/ 只读、危险命令、提交质量门)由 `.claude/settings.json` 注册的 hooks **确定性执行**,不依赖提示词自觉。 + +### Phase 1: 规划与设计 +1. 涉及**公共 API、端口签名、架构边界、新子系统**的变更**必须**调用 `brainstorming`(产出 2-3 备选方案+权衡)并经**人类确认**后实施;其余任务自判(判据: 是否改变库对下游的承诺)。动手前查阅 `research-wiki/`(单一事实源)。 +2. 功能产生运行时数据时**必须**调用 `structured-logging`。 +3. **里程碑级/跨多文件**功能编码前**必须**调用 `writing-plans`;小改动自判。审核门控: design 走 Claude 自审 → Codex 审 → **人类审**;plan 走 Claude 自审 → Codex 审 → 直接执行。 + +### Phase 2: 执行与验证 +1. **测试结果门**: 合并前每个行为变更必须有"先失败后通过"的测试证据(`test-driven-development`);bug 修复必带回归测试;不规定中间怎么走。 +2. **独立验证**: 里程碑级/跨多文件/合并前**必须**派全新上下文的 verifier subagent(`verification-before-completion`);任何规模的完成声明都必须逐条对应本会话内的工具输出(证据化声明,禁止虚报)。 +3. **反 gold-plating**: 不做任务外的重构、抽象与"顺手清理"。 + +## 4. 核心规则 + +### 4.1 核心原则(按优先级) +- **P1 YAGNI**: 不写当前用不到的代码;但并发控制、防御校验、可观测埋点、错误隔离与测试是"当前需要",不在削减之列。 +- **P2 高可读性**: 领域术语命名;注释解释"为什么"。 +- **P3 单一职责**: 一句话说不清职责(需要"和")= 拆分。 +- **P4 显式优于隐式**: 公共函数完整类型注解;依赖注入,不从全局偷取;严禁默认参数掩盖关键逻辑。 +- **P5 防御性与安全性**: 一切外部输入(网关响应、LLM 返回、配置)校验后使用;严禁 `except Exception: pass`;严禁默认值掩盖错误;敏感信息只走 `.env`。 +- **P6 可测试性**: 纯函数优先;外部依赖经 Protocol 注入;测试用真实样本或其二次构造。 +- **P7 架构依赖规则**: 决策逻辑与状态存储分离(中间件算法一份,后端可插拔);`ports.py`/`types.py`/`errors.py` 为最内层,不 import 任何具体实现;`middleware/` 只依赖端口;`transports/`、`backends/`、`telemetry/` 只实现端口,互不依赖(import-linter 契约执法)。 + +### 4.2 库铁律(本项目特有,违反即 bug) + +| 铁律 | 内容 | +|---|---| +| 零业务假设 | 库内禁止出现任何下游业务领域词汇(视频/文书/超声等)与业务 fixtures;扩展点一律 Protocol | +| 纯 asyncio 中立 | 无全局状态、无框架假设、无模块级单例;同一 `GatewayClient` 在 arq worker 与裸脚本中行为一致 | +| 取消可穿透 | `asyncio.CancelledError` 永不捕获吞没;重试循环、限流等待、流式读取全部可被取消;in-flight 资源在 finally 释放 | +| 错误分类驱动 | 一切失败必须落入 `errors.py` 四分类(Transient/SourceDead/RequestRejected/ResultInvalid),由分类决定重试/换源/熔断,禁止散落 ad-hoc 判断 | +| 遥测必录 | 每次调用(含缓存命中、失败)必经 `TelemetryRecorder` 记录,遥测写失败降级不冒泡;遥测调用点收敛为单一 helper,禁止复制参数列表(三项目 4 处复制的教训) | +| 降级方向 | 缓存/遥测后端不可用 → 静默降级(warning);限流/熔断后端不可用 → **报错而非放行**(防击穿网关) | +| 依赖极简 | 核心仅 httpx + pydantic;新增任何依赖必须进 optional extras 并经人类确认 | +| 无缓存毒化 | 缓存 key 必含 model + messages 摘要 + 命名空间/租户 + salt;多模态 content 先摘要再 hash | + +### 4.3 代码开发规范 +- 模块/类/方法必须有**中文 Docstring**;复杂逻辑用 `# Phase N` 注释组织。 +- 校验分层: 外部输入校验用显式异常(Python `-O` 移除 assert,禁止 assert 承担生产校验);assert 仅用于内部不变量。 +- 类名 `PascalCase`,私有前缀 `_`;导入顺序标准库→第三方→项目内。 +- 日志统一 **loguru**,禁用 `print()`;返回类型用 frozen dataclass。 +- 不考虑向后兼容,直接修改原文件。**例外**: `LLMResponse` 等已被三项目消费的公共类型,字段只增不删不改名(迁移兼容约束,见 ARCHITECTURE.md §5.1)。 + +### 4.4 Git 工作流 +- 一切开发在 feature 分支,严禁直改 main;频繁语义化提交;提交**必须**调用 `commit` skill;大改动前先提交回滚点。 + +### 4.5 配置管理 +- 工程配置走 `pydantic-settings` + `.env`(模板 `.env.example`,敏感项不提交);严禁硬编码默认值;缺失关键配置直接报错。 +- 多源命名约定 `{SCOPE}__{PROVIDER}__{N}__{FIELD}`;韧性参数键名沿用三项目习惯(`LLM_TIMEOUT` 等),降低迁移成本。 +- 装配只有两条路: `GatewayClient.from_env()`/`from_settings()`(工厂)或构造函数全量注入(测试/高级);库内部任何组件不得自读环境变量。 + +### 4.6 测试组织 +- `tests/{unit,integration,e2e}`;真实场景优先(录制的真实网关响应二次构造优于凭空 mock)。 +- 覆盖率目标 80%;并发/韧性行为是一等测试对象: 重试穿透取消、熔断开路半开、限流结算退款、Redis 掉线降级方向、缓存 key 隔离。 +- Redis 相关测试用真实 Redis(integration),不 mock Lua 行为;限流契约测试随实现一起交付(参考 CHSAnalyzer `tests/contracts_limiter.py`)。 +- 涉及真实 LLM 的测试输出结构化 Markdown 至 `tests/outputs/<module>/<test>_<ts>.md`。 + +## 5. 项目结构 + +```text +project_root/ +├── src/polygateway/ # 库本体(结构见奠基设计 §7: ports/types/errors 内核 + +│ # middleware/ transports/ backends/ telemetry/ structured/, +│ # 见 ARCHITECTURE.md §8) +├── tests/ # unit/integration/e2e + 限流契约测试 +├── reference/ # 三个参考项目(只读,勿改,不提交) +├── tools/ # 独立工具脚本(不被 import) +├── scripts/ # 仅 .sh +├── research-wiki/ # 单一事实源(designs/plans/findings/adrs/reviews) +├── data/ logs/ # 运行产物,不提交 +└── Makefile / pyproject.toml / .env(.example) / CLAUDE.md +``` + +> 注: 以上为**目标结构**。当前已落地: `research-wiki/ARCHITECTURE.md`、`reference/`、`.claude/`(18 个 skill 已完成 Fable 5 适配改造 + hooks 硬边界 + settings.json);其余随 M1 里程碑创建(ARCHITECTURE.md §12)。 + +硬性规则: 根目录不得出现 `.py`;`scripts/` 只放 `.sh`;禁止 `helpers/ common/ shared/ misc/ lib/` 目录名;`data/`、`logs/`、`tests/outputs/` 不提交;`reference/` 只读且不入库。 + +## 6. 上下文导航 + +| 需求 | 路径 | +|---|---| +| 架构全貌: 决策 D1-D12 及讨论过程、端口清单、错误分类、子系统设计、迁移验收 | `research-wiki/ARCHITECTURE.md`(单一事实源) | +| 功能设计文档(每次实现新功能时新增) | `research-wiki/designs/` | +| 实现计划 | `research-wiki/plans/` | +| 治理网关参考实现 | `reference/Video-Tree-TRM5/adapters/`(llm/breaker/streaming/redis_cache/telemetry) | +| 分布式限流/熔断参考实现 | `reference/CHSAnalyzer/app/coordination/`(limiter+Lua/provider_gate)与 `app/providers/governance.py` | +| 错误分类参考 | `reference/CHSAnalyzer/app/domain/errors.py` | +| OCR 两端点参考 | `reference/Video-Tree-TRM5/adapters/ocr.py` 与 `reference/CHSAnalyzer/app/providers/invokers.py:408-552` | +| 第一次抽库尝试(教训与蓝本) | `reference/GovDoc-SaaS/packages/docagent-core/` | + +## 7. 输出规范 +- 所有输出**中文**;文档优先表格/伪代码/Mermaid;禁止超 15 行代码块入文档、禁止连续超 5 条碎片列点;设计 ≤400 行、计划 ≤1000 行、报告 ≤300 行。 +- **例外**: `research-wiki/ARCHITECTURE.md` 不受行数限制——它的准绳是"后来的 AI/人类无需还原原始讨论即可准确理解全部决策及理由",宁详勿略(详细 ≠ 琐碎:记录论证与取舍,不堆砌实现细节)。 + +## 8. Skill 使用规则 + +> [!CRITICAL] +> **Skill 的 description 即触发边界。** 下表标注 **MANDATORY** 的情形是硬门(设计人类门、合并前测试证据门、合并前独立验证门、commit 格式),不得跳过;其余情形按 description 边界自判——自判看任务实质(规模/风险/是否触及公共承诺),不是省事。任何 skill 流程不得引入任务外的重构或抽象。优先级: 用户显式指令 > Skill 详细流程 > 本文件宏观规则。 + +| Skill | 触发边界 | +|-------|---------| +| `brainstorming` | **MANDATORY**: 公共 API/端口签名/架构边界/新子系统变更;其余自判 | +| `writing-plans` | **MANDATORY**: 里程碑级/跨多文件功能;小改动自判 | +| `test-driven-development`(测试结果门) | **MANDATORY**: 合并前——行为变更须有先失败后通过的测试证据 | +| `verification-before-completion`(独立验证) | **MANDATORY**: 声称完成/合并前;里程碑级须派全新上下文 verifier subagent | +| `commit` | **MANDATORY**: 提交代码时 | +| `requesting-code-review` / `receiving-code-review` | **MANDATORY**: 合并/PR 前 / 收到审查反馈后;中途审查自判 | +| `systematic-debugging` | 遇到 bug、测试失败、异常行为时(根因先于修复) | +| `structured-logging` | 功能会产生运行时数据时 | +| `subagent-driven-development` | 执行大型已批准计划时的**可选**执行器(内含合并前一次整分支审查) | +| `finishing-a-development-branch` | 实现完成且测试通过,准备集成时 | +| `using-git-worktrees` | 需要并行/隔离的分支开发时 | +| `research-wiki` / `graphify` | 管理知识库时 / 已建图后的代码结构检索 | +| 科研类: `harness-eval` `idea-creator` `novelty-check` `research-lit` | 惰性资产,现阶段不用,由模型按任务实际需要启用 | diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..a7d8e8d --- /dev/null +++ b/Makefile @@ -0,0 +1,31 @@ +.PHONY: install test lint format check ci wiki + +ENV := PolyGateway + +install: + conda run -n $(ENV) pip install -e ".[redis,postgres,structured,dev]" + +test: + conda run -n $(ENV) pytest tests/ --cov=src/polygateway --cov-report=term-missing + +# lint-imports 在 M1 模块落地前门控跳过（契约见 pyproject.toml [tool.importlinter]） +lint: + conda run -n $(ENV) ruff check src/ tests/ --fix + @conda run -n $(ENV) python -c "import polygateway.ports" 2>/dev/null \ + && conda run -n $(ENV) lint-imports \ + || echo "import-linter: M1 模块未创建，跳过（契约已在 pyproject.toml 声明）" + +format: + conda run -n $(ENV) ruff format src/ tests/ + +check: + conda run -n $(ENV) ruff format --check src/ tests/ + conda run -n $(ENV) ruff check src/ tests/ + @conda run -n $(ENV) python -c "import polygateway.ports" 2>/dev/null \ + && conda run -n $(ENV) lint-imports \ + || echo "import-linter: M1 模块未创建，跳过（契约已在 pyproject.toml 声明）" + +ci: check test + +wiki: + conda run -n $(ENV) python3 .claude/tools/research_wiki.py rebuild_index research-wiki/ diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..97b063a --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,69 @@ +[build-system] +requires = ["setuptools>=68.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "polygateway" +version = "0.1.0" +description = "PolyGateway：实验室统一的大语言模型（LLM/VLM/OCR）调度与中转库——多源、限流、重试、熔断、缓存、遥测" +requires-python = ">=3.11" +dependencies = [ + "httpx>=0.27", + "pydantic>=2.8", + "pydantic-settings>=2.4", + "loguru>=0.7", +] + +[project.optional-dependencies] +redis = ["redis>=5.0"] +postgres = ["asyncpg>=0.29"] +structured = ["json-repair>=0.28"] +sdk = ["openai>=1.30"] +dev = [ + "pytest>=8.0", + "pytest-cov>=5.0", + "pytest-asyncio>=0.23", + "ruff>=0.6", + "radon>=6.0", + "import-linter>=2.0", +] + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.pytest.ini_options] +pythonpath = ["src"] +testpaths = ["tests"] +asyncio_mode = "auto" +markers = [ + "requires_redis: 需要可达的 Redis（无则 skip）", + "requires_llm: 需要真实 LLM 网关（无则 skip，输出落 tests/outputs/）", + "slow: 慢速测试（CI 按需跑）", +] + +[tool.ruff] +target-version = "py311" +line-length = 100 + +[tool.ruff.lint] +select = ["E", "F", "W", "I", "N", "UP", "B", "A", "C4", "SIM", "TCH"] +ignore = ["E501"] + +[tool.ruff.lint.isort] +known-first-party = ["polygateway"] + +# 依赖纪律（ARCHITECTURE.md §8）：ports/types/errors/streaming 为最内层； +# middleware 只依赖端口；transports/backends/telemetry/structured/providers/sources +# 只实现端口且互不依赖；client 是唯一组装层。 +# 注：M1 模块落地前 lint-imports 由 Makefile 门控跳过。 +[tool.importlinter] +root_packages = ["polygateway"] + +[[tool.importlinter.contracts]] +name = "洋葱分层：client → 实现层 → 内核（ports/types/errors/streaming）" +type = "layers" +layers = [ + "polygateway.client", + "polygateway.middleware : polygateway.transports : polygateway.backends : polygateway.telemetry : polygateway.structured : polygateway.providers : polygateway.sources", + "polygateway.ports : polygateway.types : polygateway.errors : polygateway.streaming", +] diff --git a/research-wiki/ARCHITECTURE.md b/research-wiki/ARCHITECTURE.md new file mode 100644 index 0000000..09067ca --- /dev/null +++ b/research-wiki/ARCHITECTURE.md @@ -0,0 +1,525 @@ +# PolyGateway 架构文档 + +> **文档定位**: 本文档是 PolyGateway 的**架构单一事实源**,记录项目边界、全部架构决策(含讨论过程与被否决的备选方案)、各子系统设计与三项目迁移验收标准。 +> +> - 与 `research-wiki/designs/` 的关系: `designs/` 存放**每次实现具体功能时**的设计文档(受 ≤400 行约束);本文档**不受行数限制**,以"让后来的 AI 或人类无需还原原始讨论就能准确理解全部决策及其理由"为准绳。功能设计文档与本文档冲突时,以本文档为准,或先修订本文档。 +> - 状态: 边界与决策已与人类逐条讨论确认(2026-07-19),待人类终审。 +> - 前置输入: 对 `reference/` 下三个项目(Video-Tree-TRM5 / CHSAnalyzer / GovDoc-SaaS)LLM 层与 OCR 层的完整代码调研,关键证据以 `文件:行号` 形式散布于本文各节。 + +--- + +## 0. 一页速览 + +PolyGateway 是实验室内部统一的大语言模型(LLM/VLM/OCR,音频预留端口)调度与中转库。它治理的单位是**一次模型调用**:业务侧把就绪的 messages/图像字节交给它,它负责多源选择、限流、发请求、流式解析、错误分类、重试换源、熔断、缓存、遥测记账,返回统一的响应类型。 + +期望的使用形态(90% 用户只需要这三行): + +```python +client = GatewayClient.from_env() +resp = await client.chat(messages) # resp: LLMResponse +``` + +全部治理能力组织为**中间件洋葱**,全部易变点(状态存哪、协议怎么发、源怎么选、输出怎么解析、遥测记到哪)定义为**端口(Protocol)**,按部署配置插拔。库对下游零业务假设,验收标准是三个参考项目删掉各自的治理代码、换成本库后原测试全过(§11)。 + +--- + +## 1. 背景:为什么需要这个库 + +### 1.1 三个参考项目与它们的同源性 + +实验室三个项目各自维护一套 LLM/VLM 调用基础设施。调研证实它们本质是**同一套治理栈的三次复制与变异**: + +- **Video-Tree-TRM5**(长视频理解科研,批处理形态):治理栈的源头之一。`adapters/llm.py` 的 `GovernedLLMClient` 实现五层治理(熔断→缓存→重试+流式→写缓存→遥测)。 +- **GovDoc-SaaS**(多租户文书 SaaS,服务形态):**第一次抽库尝试**。`packages/docagent-core/src/docagent_core/llm/` 已经是独立包,代码注释多处标注"移植自 CHSAnalyzer2 / Video-Tree-TRM5"。但装配层(配置→client)始终没写完,且并发限流完全缺失。 +- **CHSAnalyzer**(超声影像分析,服务形态):独立演化出三者中**最强的分布式治理**——Redis+Lua 原子限流、跨进程熔断、多源多账号,但完全没有响应缓存,且同一项目里还并存着一个无任何治理的裸 SDK 调用(`core/eval/judge.py`,反面教材)。 + +三者共同的技术底色高度一致,这是统一库可行的基础:全部用 httpx 手写 OpenAI 兼容 SSE 协议(不用官方 SDK 发聊天请求)、自研重试(不用 tenacity)、pydantic-settings + `.env`、loguru、`@runtime_checkable Protocol` 依赖注入、组装点(Composition Root)统一构造。 + +### 1.2 能力对比矩阵 + +| 能力 | Video-Tree-TRM5 | GovDoc-SaaS | CHSAnalyzer | +|---|---|---|---| +| 治理网关(重试/退避/超时) | ✅ `GovernedLLMClient` | ✅ 同款移植 | ✅ Invoker/Governance 分层(结构最好) | +| 错误分类 | ⚠️ 二分类(瞬时/致命) | ⚠️ 同款 | ✅ 三分类 + Retry-After 解析 + 工件级失败 | +| 限流 | ❌ 仅 `asyncio.Semaphore` | ❌ 完全没有 | ✅ Redis+Lua 六道闸(并发/RPM/TPM × 全局/单源) | +| 熔断 | ⚠️ 进程内存态 | ⚠️ 同款 | ✅ Redis 跨进程 + epoch fencing | +| Redis 响应缓存 | ✅ sha256 内容寻址 | ✅ 同款(缺租户隔离) | ❌ 完全没有 | +| 多源多账号切换 | ❌ 单源 | ❌ 单源 | ✅ 多源配置 + 选源策略端口 | +| 流式活性看门狗 | ✅ TTFT/token间/总时长三层 | ✅ 同款 | ✅ 同款 | +| 多模态(图像) | ✅ base64 注入 content 数组 | ❌ 明确不做 | ✅(端口限单图,靠拼图绕过) | +| OCR | ⚠️ 裸调 `/ocr/text` | ❌(明确剥离) | ✅ 全治理 `/parse` | +| 音频/ASR | ❌ 死配置(配了 Groq whisper 无实现) | ❌ | ❌ | +| 遥测 | ✅ SQLite 每调用必录 | ✅ 同款(设计可注入 Postgres) | ⚠️ 仅 JSON 结构化日志 | +| 装配工厂(配置→client) | ⚠️ 每项目手写 `_build_adapters()` | ❌ 缺失(env 定义了但没人读) | ⚠️ 手写 container | + +**没有任何一个项目是完整的**;每个项目各有一块别人没有的能力,同时各缺一块。 + +### 1.3 每次重新开发的到底是什么(五类重复) + +1. **治理栈本体**被复制至少三次,每次复制产生变异(缓存 salt、租户隔离、错误分类粒度各不相同),修一个 bug 修不到另外两份。 +2. **"配置 → 装配 client"** 每个项目手写一遍;GovDoc 甚至 `.env` 参数定义齐全但装配层缺失。 +3. **三套互不一致的 JSON 结构化输出解析**:json_repair / 手写正则 / `find('{')`+`rfind('}')` 并存,甚至同一项目内并存(Video-Tree)。 +4. **provider 差异靠字符串猜**:`"qwen" in provider`、`model.split("-")[0]` 推断 provider,接新 provider 要改核心类。 +5. **同样的坑各踩各的**:遥测调用逐字复制 4 次(Video-Tree 与 GovDoc 都有,`record_llm_call` 十几个参数的调用散布在缓存命中/成功/致命/瞬时/非重试五个分支);CHSAnalyzer 的 judge 同步裸调无治理。 + +### 1.4 各项目关键资产索引(移植蓝本) + +| 资产 | 来源 | 移植去向(§7) | +|---|---|---| +| 治理网关主循环(参考结构,需重构掉遥测复制) | `Video-Tree/adapters/llm.py`、`GovDoc/packages/docagent-core/src/docagent_core/llm/client.py` | client + middleware | +| 三层流式活性看门狗(纯函数,近乎原样复用) | 三项目同款 `streaming.py` | `streaming.py` | +| 进程内熔断器(时钟注入、单探针) | `Video-Tree/adapters/breaker.py` | `backends/memory/` | +| Redis 跨进程熔断(epoch fencing) | `CHSAnalyzer/app/coordination/provider_gate.py` | `backends/redis/` | +| Redis+Lua 六道闸限流(含 permit/settle) | `CHSAnalyzer/app/coordination/limiter.py` + `scripts.py`(约 245 行 Lua)及契约测试 `tests/contracts_limiter.py` | `backends/redis/` | +| Redis 响应缓存(sha256 key、TTL、salt、静默降级) | `Video-Tree/adapters/redis_cache.py` | `backends/redis/` | +| 错误三分类 + HTTP→领域翻译 | `CHSAnalyzer/app/domain/errors.py`、`app/providers/invokers.py:127-227` | `errors.py` + transports | +| 治理编排泛型核心(VLM/OCR 共用一个 core 的先例) | `CHSAnalyzer/app/providers/governance.py:200` | middleware 组合 | +| SQLite 遥测(WAL、幂等、to_thread 桥接) | `Video-Tree/adapters/telemetry.py`、`GovDoc/.../telemetry_sqlite.py` | `telemetry/sqlite.py` | +| 统一响应类型 `LLMResponse`(frozen dataclass) | 三项目同款 `types.py` | `types.py`(超集兼容) | +| 多源配置解析(`SCOPE__PROVIDER__N__FIELD`) | `CHSAnalyzer/app/config.py:223` | 配置层 | +| JSON 围栏剥离 + json_repair + 变体归一化 | `Video-Tree/core/agent/loop.py:341-399` | `structured/json_repair.py` | +| MonkeyOCR 两端点 invoker | `Video-Tree/adapters/ocr.py`、`CHSAnalyzer/app/providers/invokers.py:408-552` | `transports/monkey_ocr.py` | + +--- + +## 2. 定位、目标与非目标 + +### 2.1 定位:治理单位是"一次模型调用" + +三个项目呈现两种使用形态,曾担心无法统一;结论是**在调用层天然同构,在编排层不应统一**: + +```text +CHSAnalyzer / GovDoc(服务形态) Video-Tree(批处理形态) +────────────────────────────── ───────────────────────── +HTTP API → arq 队列 → worker 协程 脚本 → asyncio.gather 协程 + └────── await client.chat(...) ──────┘ ← 库站在这一层 + (多源/限流/重试/熔断/缓存/遥测) +``` + +- **编排层**(上):单位是"一个业务任务"(解析一份文书、建一棵视频树),带任务持久化、阶段重试、租户配额等业务概念——**留在业务侧**。 +- **调用层**(下):单位是"一次模型调用"。对库而言,上方是 arq worker 里的协程还是 gather 出来的协程毫无区别,都只是并发调用者。 + +为使两种形态同构接入,库承诺三条硬约束(详见 §4.2 库铁律与 §6.4):纯 asyncio 无全局状态、`CancelledError` 全栈穿透、配额满时的行为(等待 vs 快速失败)可配置。 + +### 2.2 目标能力 + +请求封装(LLM/VLM 聊天 + OCR)、多源多账号与选源、限流(并发/RPM/TPM)、错误分类与重试退避、熔断、Redis 响应缓存、流式活性看门狗、遥测(含成本统计)、结构化输出策略、装配工厂。全组件端口化可插拔,配置驱动装配。 + +### 2.3 非目标(已确认,含理由) + +| 不做 | 理由(讨论结论) | 归属 | +|---|---|---| +| 任务队列(arq)/任务编排 | 队列单位是业务任务,库单位是单次调用,高度不同;强行进库会迫使批处理项目部署队列、并把"任务"业务概念污染进零业务假设的库。Video-Tree 声明了 arq 依赖却从未使用(死依赖)是现实佐证 | 业务侧 | +| 视频抽帧(ffmpeg)、图像裁剪/拼接/增强等预处理 | 纯业务先验(超声图表格在左上角、每 5 帧一批等),且会拖入 ffmpeg/PIL/numpy 重依赖;库只收就绪的 content 数组/图像字节 | 业务侧 | +| OCR 结果的几何映射(坐标换算/归一化/marker 推算) | 同上,业务先验;库只返回 OCR 服务的原生 bbox + page_size | 业务侧 | +| Prompt 模板管理 | 三项目组织方式差异过大(md 文件加载 / 模块常量 / 版本化进化资产),无公共形态可抽 | 业务侧 | +| Agent Loop / 工具调用编排 | 属 agent 运行时,不属调用治理;但其中的 JSON 解析部分下沉为库的结构化输出策略(D7) | 业务侧 | +| 音频模型**实现** | 实际用量少,收益不及维护成本;**只预留端口**,需求到来时增量实现 | 库(仅端口) | +| 公平调度(CHSAnalyzer 的按 Session 轮转 position_scheduler) | 调用级公平队列是该项目的业务需求,不是通用治理关注点 | 业务侧 | + +--- + +## 3. 架构决策记录(D1–D12,含讨论过程与备选方案) + +> 每条决策记录格式:**决策 / 背景与讨论 / 被否决的备选 / 影响**。这些决策已与人类逐条确认;推翻任何一条需要人类批准并修订本节。 + +### D1 架构风格 = 端口适配器 + 中间件洋葱模型 + +**决策**: 借鉴 Clean Architecture 的三条原则——依赖规则(核心不依赖具体技术)、端口与适配器(Protocol 定义接缝)、组装点(所有构造集中注入);**不照搬**其面向应用的四层分层(Entities/Use Cases/Interface Adapters/Frameworks)。库内部的组织模式采用**中间件洋葱**(同 ASGI middleware / gRPC interceptor / Rust tower):重试、限流、熔断、缓存、遥测各为一层,层与层正交,顺序与取舍是配置。 + +**背景与讨论**: 人类提问"是否借鉴《Clean Architecture》,是否有更好的指导思想"。结论:那本书为应用程序而写,库没有"用例层",硬套四层会造出空转抽象。对库更适配的思想来源: +- **Hexagonal / Ports & Adapters**(Cockburn):三项目已在实践的本质。 +- **《A Philosophy of Software Design》(Ousterhout)的"深模块、窄接口"**:接口复杂度是用户付的成本。落地为——90% 用户三行起步(`from_env()` → `chat()`),全部可配置性经构造函数暴露给需要的人,但绝不强迫简单用户理解。 +- **中间件洋葱**:与治理栈天然同构。反面证据:三项目的 `GovernedLLMClient.chat()` 是约 500 行的方法,五层治理手工内联在一个重试循环里,横切关注点没有被切开,遥测调用因此被迫复制 4 次。洋葱模型下遥测就是一层,只写一次。 + +**被否决的备选**: 照搬应用四层(过度抽象);继续单体 God-method(现状,已证明产生复制)。 + +**影响**: `Middleware` 协议为 `(request, call_next) -> response`;§4 给出默认层序及理由;§8 的依赖纪律由 import-linter 机械执法。 + +### D2 Transport 默认手写 httpx OpenAI 兼容协议;`Transport` 是端口,SDK 适配器可选 + +**决策**: 默认 transport 复用三项目久经实战的手写 httpx + SSE 实现;同时把 transport 定义为端口,提供可选的 `OpenAISDKTransport`(薄封装),未来接 Anthropic 原生/Gemini 时增量加对应 transport,治理栈零改动。 + +**背景与讨论**: 人类要求完整阐述官方 SDK 与手写的差异优劣。核心对比: + +| 维度 | 手写 httpx | 官方 SDK(openai) | +|---|---|---| +| SSE 协议解析(帧格式、畸形帧、usage 帧、[DONE]) | 自己写自己修(约 200 行),但全可控 | SDK 维护,跟随协议演进 | +| 错误分类 | 状态码 + body 字符串匹配,自己写 | 类型化异常层级(RateLimitError 等),映射干净 | +| 非标字段(qwen `enable_thinking`、deepseek `reasoning_content`) | 天然支持 | `extra_body` 写入 + `model_extra` 读出,**够用** | +| 流式活性看门狗 | 天然支持 | **同样支持**(看门狗包的是 `__anext__`,对 SDK 的 chunk 迭代器同样生效) | +| 线路级异常定性(如"未收到 [DONE] 即断流") | 能精确检测并作为熔断输入 | **做不到**(SDK 内迭代器正常结束)——这是换 SDK 的真实损失 | +| 依赖 | 零新增 | openai 包,有大版本破坏史 | +| 私有网关不合规帧 | 实测调,全可控 | SDK 行为不受控 | +| 非 OpenAI 协议 | 每协议再手写一套 | 每协议换官方 SDK,成本低 | + +讨论中澄清了两个常见误解:(a)"用 SDK 就失去 thinking/自定义参数"——不成立,`extra_body`/`model_extra` 是官方逃生口;(b)"用 SDK 就没法做活性看门狗"——不成立,看门狗包装异步迭代器,与实现无关。**真正拿不回来的只有线路级异常的精确定性**,而该信号目前是熔断器输入之一。 + +默认选手写的理由:实验室主要打 OpenAI 兼容私有中转网关(newapi 等),该场景下 SDK 增益最小(类型化错误)、风险最大(网关不合规帧);且现有代码已实战验证。 + +**被否决的备选**: 全库押注 SDK(丢线路级信号,依赖风险);全库永远手写(接非 OpenAI 协议成本高)。端口化让"SDK vs 手写"从全局站队降级为 per-provider 选择。 + +**影响**: `Transport` 端口职责 = 请求体组装 + 流式解析 + HTTP/线路错误 → 领域错误翻译(§6.2);provider 差异不写在 transport 里,写在 provider 注册表(D11)。 + +### D3 限流/熔断双后端:决策逻辑一份,状态存储端口化 + +**决策**: 限流与熔断的**算法**(退避公式、窗口计算、开路/半开/探针状态机)只实现一次;**状态存储**(计数器、租约、失败数)定义为端口,发货 `InMemory*` 与 `Redis*` 两组实现,按部署配置选择。 + +**背景与讨论**: 人类初期"限流/熔断状态放哪一层没想好"。分析:状态放进程内存——单进程批处理(Video-Tree)完全正确、零依赖零开销;但多 worker 时每进程各持一份计数器,配置 RPM=60 会实际打出 60×N,熔断信息也不共享。状态放 Redis(CHSAnalyzer 现状)——全局限额真实生效,但强依赖 Redis + Lua,对单进程脚本过重。结论:**这不是架构二选一,而是不同部署形态各有正确答案**;把状态存储做成端口后,该问题从"现在必须决定、以后改要动核心"降级为"每个项目接入时的一行配置"。人类确认接受此方案。 + +**被否决的备选**: 只做 Redis(强迫脚本用户起 Redis);只做进程内(CHSAnalyzer 无法迁移);在库外让各项目自己解决(重复即回归)。 + +**影响**: §7.3/§7.4 分别定义两种后端必须满足的同一套语义契约;后端不可用时的降级方向见 §4.2 库铁律(限流/熔断后端不可用必须**报错而非放行**)。 + +### D4 Redis 响应缓存为第一优先交付 + +**决策**: 继承 Video-Tree/GovDoc 已验证的方案(sha256 内容寻址 key、TTL、Redis 不可用静默降级),并修正三个已知缺陷:key 增加**命名空间/租户**字段(GovDoc 多租户铁律,现实现缺失,存在跨租户缓存命中风险)、**多模态 content 先摘要再 hash**(Video-Tree 现把整段 base64 图片喂进 sha256,开销巨大)、保留 **cache salt**(跨 epoch 强制重采样)。 + +**背景与讨论**: 人类明确"Redis 缓存对我们很重要,应该实现"。澄清过一个误会:D3 的"双后端"只关于限流/熔断状态,与响应缓存无关;缓存后端本身也是端口(`CacheBackend`),但 Redis 实现是默认且第一优先。CHSAnalyzer 完全没有响应缓存(相同图像+指令重复付费),迁移后免费获得。 + +**影响**: key 组成公式见 §7.5;缓存命中也必须写遥测(cache_hit=True,latency_ms=0),且不消耗限流配额(§4.3 层序理由)。 + +### D5 arq/任务队列不进库 + +**决策与理由**: 见 §2.1 与 §2.3 第一行。人类确认。 + +**影响**: 库承诺三条约束(纯 asyncio 中立、取消穿透、配额满行为可配)保证两种形态同构接入;附带交付 `gather_bounded(calls, concurrency)` 十行级便利函数,替代 Video-Tree 手搓的 semaphore+gather 样板——它是便利函数,不是队列。 + +### D6 多源多账号是硬需求 + +**决策**: 多源配置(每源独立 base_url/api_key/model/限额)+ `SourceSelector` 端口(首发 round_robin 与 least_inflight 两策略)+ 错误分类驱动换源(§6)。人类确认"必须满足"。 + +**影响**: 单源项目(GovDoc/Video-Tree 现状)= 源列表长度为 1 的特例,不感知复杂度;源冷却备忘(熔断源在本地记冷却截止,避免白烧配额去探测)一并移植(`CHSAnalyzer governance.py:107`)。 + +### D7 结构化输出为可选策略,不二选一 + +**决策**: `StructuredOutputStrategy` 端口,两个首发实现:`JsonRepairStrategy`(prompt 约定 + 围栏剥离 + json_repair 事后修复 + provider 变体归一化,即三项目现状的收敛统一)与 `NativeSchemaStrategy`(response_format / function calling,网关支持时使用)。按 provider 能力声明(D11)与调用参数逐调用选择。人类明确要求做成可选项。 + +**影响**: 消灭三项目三套互不一致的 JSON 解析(§1.3 第 3 条);解析失败抛 `ResultInvalidError`(坏结果≠坏服务,§6.3)。 + +### D8 遥测必录、后端可插拔、成本并入遥测 + +**决策**: 每次调用(含缓存命中与失败)必经 `TelemetryRecorder` 记录;字段继承三项目 15 字段规范(§7.8)并新增成本字段;后端首发 SQLite(默认)与 Postgres;pricing 表(model → 单价)把 token 用量换算为金额。人类确认遥测方式需可配置、成本统计加入遥测。 + +**影响**: 遥测调用点收敛为单一 helper(针对三项目 4 处复制的教训,列为库铁律);遥测写失败降级不冒泡(记录基础设施不得拖垮业务调用)。 + +### D9 OCR 提前纳入;端口族而非单接口 + +**决策**: OCR 优先级提前(高于音频)。因两个项目调用**同一套 MonkeyOCR 服务的两个不同端点、两种输出语义**,统一为单接口会强行合并不同类型的能力,故做成端口族:`OcrTextPort.recognize_text(bytes)`(对应 `/ocr/text`,纯文本转录)与 `OcrLayoutPort.parse_layout(bytes)`(对应 `/parse`,ZIP→结构化元素+bbox)。OCR 调用走与 LLM 同一套中间件治理栈(CHSAnalyzer 已有 VLM/OCR 共用泛型治理核心的先例)。 + +**背景(调研结论)**: Video-Tree 用 `/ocr/text` 把帧文字作为"硬证据"并置进 VLM 提示词,**裸调**(无重试/限流/熔断,单帧失败跳过,手写双端点轮询)——迁移后免费升级为全治理。CHSAnalyzer 用 `/parse` 定位表格 bbox,已全治理。GovDoc 无 OCR 且是明确架构否决(旧系统被"OCR 挂死"拖垮,教训反哺 §6.4 取消语义)。多后端预留有直接证据:CHSAnalyzer provider 白名单已含 `glm`(延后),设计文档明确三种后端图片输入形态各异(VLM=base64、Monkey=multipart、GLM=URL)——因此端口收 `bytes`,编码差异封装在 invoker 内部。 + +**影响**: 协议细节与后处理分界见 §7.10;`TableLocator` 这类业务化端口留在业务侧,由库端口结果组装。 + +### D10 音频只留端口 + +**决策**: `AudioPort` 占位 Protocol,不实现。人类确认"留出接口即可"。Video-Tree 的 ASR 死配置(`.env` 配了 Groq whisper 但零实现)不作为需求证据。协议形态(OpenAI `/audio/transcriptions` vs 聊天式多模态)留待真实需求出现时定,届时写功能设计文档。 + +### D11 provider 差异显式化(注册表) + +**决策**: 消灭 `"qwen" in provider`、`model.split("-")[0]` 式字符串猜测。显式 provider 注册表,每个 provider 声明:thinking 参数注入方式(deepseek `{"thinking":{"type":"enabled"}}` / qwen `{"enable_thinking": True}`)、思考流字段(`reasoning_content` / `<think>` 标签剥离)、原生 schema 能力(供 D7 策略选择)、默认错误翻译细则。新 provider = 注册一个条目,不改核心类。 + +### D12 零业务假设 + 单向依赖(继承 GovDoc 铁律) + +**决策**: 库内禁止出现任何下游业务领域词汇(视频/文书/超声等)与业务 fixtures;扩展点一律 Protocol;import-linter 契约机械化执法(§8)。GovDoc 已证明这套纪律可执行(`pyproject.toml [tool.importlinter]`)。 + +--- + +## 4. 总体架构 + +### 4.1 洋葱结构 + +```mermaid +flowchart TB + subgraph 业务侧["业务侧(三项目各自保留)"] + A1["arq worker / FastAPI<br/>(CHSAnalyzer, GovDoc)"] --- A2["批处理脚本 asyncio.gather<br/>(Video-Tree)"] + A3["抽帧 / 图像预处理 / prompt 组装 / Agent Loop / 公平调度"] + end + subgraph PolyGateway["PolyGateway: GatewayClient"] + M1[TelemetryMW 遥测+成本] --> M2[CacheMW 响应缓存] + M2 --> M3[BreakerMW 熔断] + M3 --> M4[RateLimitMW 限流] + M4 --> M5[RetryMW 重试+退避+换源] + M5 --> T["Transport 端口<br/>OpenAICompat(httpx,默认) / OpenAISDK(可选) / MonkeyOCR"] + end + subgraph 状态后端["可插拔状态后端"] + B1["InMemory*(单进程)"] + B2["Redis*(跨进程)"] + B3["SQLite / Postgres 遥测"] + end + 业务侧 -->|"await client.chat(...) / ocr.recognize_text(...) / ocr.parse_layout(...)"| PolyGateway + M1 -.-> B3 + M2 & M3 & M4 -.-> B1 & B2 +``` + +分层要义:**决策逻辑(中间件算法)只有一份;易变处全部是端口**——状态存哪(后端)、协议怎么发(transport)、源怎么选(selector)、输出怎么解析(structured strategy)、账记到哪(telemetry)。 + +### 4.2 库铁律(与 CLAUDE.md §4.2 同步维护,冲突以 CLAUDE.md 为准) + +零业务假设 / 纯 asyncio 中立(无全局状态、无框架假设、无模块级单例)/ 取消可穿透 / 错误分类驱动(禁止 ad-hoc 判断)/ 遥测必录且调用点收敛单一 helper / 降级方向(缓存与遥测后端挂 → 静默降级;限流与熔断后端挂 → **报错而非放行**,防击穿网关)/ 依赖极简(核心仅 httpx+pydantic,其余 optional extras)/ 缓存 key 防毒化。 + +### 4.3 默认中间件层序及理由(外→内) + +**遥测 → 缓存 → 熔断 → 限流 → 重试 → transport** + +| 相对顺序 | 理由 | +|---|---| +| 遥测最外 | 观测一切,包括缓存命中与各类失败;任何路径都留痕 | +| 缓存在熔断/限流外 | 缓存命中不应消耗限流配额,也不应被开路的熔断挡住(命中不打网关) | +| 熔断在限流外 | 开路时直接拒绝,不占用限流租约、不排队等配额 | +| 重试在限流内 | 每次重试是一次真实网络请求,必须重新过限流闸;否则重试风暴击穿配额。此顺序意味着限流后端看到的是"含重试的真实请求数" | +| 换源在重试循环内 | `TransientError`/`SourceDeadError` 触发选下一源(§6),同一逻辑调用的多次尝试可落在不同源上 | + +层序与取舍最终是配置——项目可增删层(如无 Redis 环境去掉 CacheMW),但改默认顺序需理解上表理由。 + +### 4.4 一次调用的生命周期(walkthrough) + +1. **缓存命中**: TelemetryMW 记录(cache_hit=True, latency_ms=0)→ CacheMW 返回,不触达任何更内层。 +2. **正常路径**: 穿过熔断(闭路)→ 限流 acquire permit(并发/RPM/TPM 三闸,token 预扣)→ RetryMW 首次尝试 → selector 选源 → transport 发请求、流式解析(看门狗包裹)、收 usage 帧 → permit 按实际 usage settle(多退少补)→ 回程写缓存 → 遥测记成功(含 ttft/max_inter_token/成本)。 +3. **瞬时错误**(超时/5xx/429/SSE 异常): transport 翻译为 `TransientError` → RetryMW 指数退避+jitter(取 Retry-After 提示与退避的较大值)后换源重试;每次尝试独立 call_id、独立过限流闸、失败即报熔断计数与遥测。 +4. **源死亡**(401/403/欠费): `SourceDeadError` → 该源熔断 force_open + 本地冷却备忘 → 立即换下一源,不退避等待。 +5. **请求被拒**(400/坏输入): `RequestRejectedError` → 不重试不换源,直接上抛;遥测记录。 +6. **开路/全源耗尽**: `CircuitOpenError` / `AllSourcesExhausted` → 按配置 wait(等待恢复,含 stall 判定)或 fail-fast 上抛。 +7. **任意时刻取消**: `CancelledError` 穿透所有层;in-flight permit 与连接在 finally 释放。 + +--- + +## 5. 核心类型 + +### 5.1 `LLMResponse`(frozen dataclass,与三项目超集兼容) + +**兼容约束(硬)**: 以下字段为三项目现有消费面,只增不删不改名: + +| 字段 | 类型 | 说明 | +|---|---|---| +| `content` | str | 正式输出文本 | +| `thinking` | str | 思考流内容(reasoning_content / think 标签,按 provider 注册表提取) | +| `model` / `provider` | str | 溯源 | +| `prompt_tokens` / `completion_tokens` | int | usage 帧读取;缺失时按估算标注 | +| `latency_ms` | int | 总延迟 | +| `ttft_ms` / `max_inter_token_ms` | float? | 流式活性测量 | +| `cache_hit` | bool | 是否缓存命中 | +| `call_id` | str | UUID,每次**尝试**独立 | + +新增字段(库扩展): `source_name`(多源溯源)、`cost`(pricing 换算,可为 None)、`usage_source`(measured/estimated)。 + +### 5.2 其他类型 + +`ChatRequest`(model/messages/结构化输出参数/per-call 覆盖项)、`Usage`(tokens + elapsed,OCR 无计费填 0)、`OcrTextResult`(text + 溯源三件套 source_name/usage/raw)、`OcrLayoutResult`(elements 含 bbox/type + page_size + 溯源)。全部 frozen dataclass。空结果语义:合法"无内容"用空值/None 表达,调用失败必须走异常——二者严格区分。 + +--- + +## 6. 错误模型 + +### 6.1 统一四分类 + 熔断信号(融合 CHSAnalyzer 三分类与 GovDoc 二分类) + +| 错误类 | 触发 | 重试 | 换源 | 熔断计数 | +|---|---|---|---|---| +| `TransientError` | 超时/5xx/429/网络抖动/SSE 异常(畸形帧、断流无 [DONE])/看门狗超时 | ✅ 退避后 | ✅ | ✅ | +| `SourceDeadError` | 401/403/欠费/insufficient_quota(429 body 细分) | ❌ | ✅ 立即 | ✅ force_open | +| `RequestRejectedError` | 400/请求格式错/坏输入(如不支持的图像格式) | ❌ | ❌ | ❌ | +| `ResultInvalidError` | 调用成功但内容不可解析(JSON 修不好、ZIP 缺关键文件) | ❌(策略层可选二次尝试) | ❌ | ❌(熔断记**成功**) | +| `CircuitOpenError` / `AllSourcesExhausted` | 开路 / 全源耗尽 | 调用方决定: wait / fail-fast 可配 | — | — | + +### 6.2 翻译规则(transport 层职责) + +| 输入 | 翻译为 | +|---|---| +| httpx Timeout/Transport 错误、`StreamLivenessTimeout`、SSE 异常 | `TransientError` | +| HTTP 429(body 无 insufficient_quota)、500/502/503/504 | `TransientError`(携 `Retry-After` 解析值,仅支持秒数形态) | +| HTTP 429 + body 含 insufficient_quota、401、403 | `SourceDeadError` | +| HTTP 400 | `RequestRejectedError` | +| 解析层失败(结构化输出/OCR ZIP) | `ResultInvalidError` | + +### 6.3 "坏结果 ≠ 坏服务"(ResultInvalidError 语义,继承 CHSAnalyzer) + +由输入内容决定的**确定性失败**(这张图就是解析不出表格、这段输出就是修不成 JSON):服务是健康的,换源重试只会白烧配额。因此熔断器记成功、不换源、异常上抛消耗业务侧的失败预算。出处:`CHSAnalyzer governance.py:237-239`。 + +### 6.4 取消语义 + +`asyncio.CancelledError` 永不捕获吞没(它继承 BaseException,防御性 `except Exception` 天然放行,但库内严禁 `except BaseException` 与裸 `except:`)。重试循环、限流等待、退避 sleep、流式读取全部可被取消;in-flight permit、httpx 连接在 finally 释放。教训出处:GovDoc 旧系统"OCR 挂死、前端无取消能力"(`GovDoc research-wiki/designs/2026-07-09-docagent-core-monorepo-design.md:16`)。 + +--- + +## 7. 子系统设计 + +### 7.1 Transport + +**职责**: 一次原始调用的全部协议细节——请求体组装(含 provider 注册表注入的 thinking 参数)、发送、流式 SSE 解析(增量 content/reasoning_content、usage 帧、[DONE] 检测)、HTTP/线路错误按 §6.2 翻译。**不含**重试/限流/缓存(那是中间件的事)。 + +- `OpenAICompatTransport`(默认): httpx.AsyncClient(每源一个,预配 Authorization 与分段超时),SSE 解析移植三项目的模块级纯函数;强制 `stream_options.include_usage`。**非流式快路径**: 短请求可配 `stream=False`(三项目都写死 stream=True 强迫短请求走 SSE+看门狗,库放开)。 +- `OpenAISDKTransport`(可选 extra): 薄封装,`max_retries=0` 关掉 SDK 自带重试(治理归中间件),`extra_body`/`model_extra` 通道非标字段。 +- `MonkeyOcrTransport`: 见 §7.10。 + +### 7.2 重试与退避 + +指数退避 + jitter: `delay = min(base * 2^attempt, max_delay) * uniform(0.5, 1.5)`,与 `Retry-After` 提示取较大值。**单层重试原则**: 库内只有 RetryMW 一层重试;Video-Tree/GovDoc 现存的"治理层重试 + Agent 步级二次重试"双层结构(异常集合不同、职责重叠)不复制——业务侧如需任务级重试,自行在库外包,且语义是"任务重试"不是"调用重试"。 + +### 7.3 限流 + +**语义契约**(两后端同一套): `try_acquire(source, est_tokens) -> Permit | 拒绝(含等待提示)`;`Permit.settle(actual_usage)` 按实际 usage 结算(预扣保守入场,多退少补);`Permit.release()` 在 finally 必然执行;permit 带 TTL 租约防进程死亡泄漏。 + +- `RedisLimiter`: 移植 CHSAnalyzer 六道闸——单条 Lua 原子检查全局并发/单源并发(ZSET 租约)/全局 RPM/单源 RPM/全局 TPM/单源 TPM;窗口 id 用 **Redis 服务器时钟**(TIME 命令)统一多进程口径。随实现移植契约测试。 +- `InMemoryLimiter`: 同一契约的进程内实现(semaphore + 滑动窗口计数);单进程场景下语义等价。 +- **配额满行为可配**: `wait`(等待,配 stall 判定——本地等待超窗 + 全局无进展超窗双条件才判卡死)或 `fail-fast`(立即抛)。 +- 全局活性信号: `mark_progress()`/`progress_age_s()`("最近一次出餐"时刻)供背压 stall 判定,移植 `CHSAnalyzer limiter.py:193`。 + +### 7.4 熔断 + +**状态机**(算法一份): 闭路 --连续失败达阈值--> 开路(冷却)--冷却到期--> 半开(只放**一个**探针,防惊群)--成功--> 闭路 / --失败--> 开路。`force_open` 支持 SourceDeadError 一击即熔。按 source_name 分别计数。 + +- `InMemoryBreakerState`: 移植 Video-Tree `breaker.py`(时钟由调用方注入,纯确定性可测)。 +- `RedisBreakerState`: 移植 CHSAnalyzer `provider_gate.py`,含 **epoch fencing**(防旧世代进程污染新状态)。 +- 阈值指导: 有效阈值 = `max(configured_threshold, concurrency * 2)`(三项目 .env 注释中的手动约定,库内自动计算)。 +- 源冷却备忘: 开路源在进程本地记冷却截止时刻,选源时跳过,避免白烧 RPM 去探测(移植 `governance.py:107`)。 + +### 7.5 响应缓存 + +**key 公式**: `sha256(canonical_json({model, messages_digest, namespace, salt}))`,前缀 `pgw:cache:`。 + +- `messages_digest`: 文本部分原文参与;多模态 content part(base64 图像等)先各自 sha256 摘要再参与——修正 Video-Tree 把整段 base64 进 hash 的开销问题,且 key 稳定性不变。 +- `namespace`: 必填(项目名/租户 id),修正 GovDoc 缓存 key 缺租户隔离与多项目共用 Redis 时的互相毒化风险。 +- `salt`: 可选,跨 epoch 强制重采样(Video-Tree 需求)。 +- value = `LLMResponse` 的 JSON;TTL 必填且 > 0(禁止永不过期,继承 Video-Tree 校验);Redis 不可用 → get 返回 None、set 吞异常记 warning(静默降级)。**只缓存成功响应**;`ResultInvalidError` 的原始响应不缓存(避免固化坏结果)。 + +### 7.6 流式活性看门狗 + +三层超时——TTFT(首 token)/ inter_token(token 间隔)/ total(总时长),分别抛出携超时类别的 `StreamLivenessTimeout`(归 `TransientError`)。实现为领域无关纯函数,只包裹迭代器的单次 `__anext__`,用 `asyncio.timeout` 的 `expired()` 区分本层 deadline 与上游超时,避免取消泄漏。三项目同款,近乎原样移植。约束 `0 < inter_token < ttft < timeout_s`(继承 CHSAnalyzer SourceConfig 不变式校验)。 + +### 7.7 多源与选源 + +`SourceConfig`: name/provider/base_url/api_key/model/超时组/限额组(单源并发/RPM/TPM)/enable_thinking。聚合自环境变量 `{SCOPE}__{PROVIDER}__{N}__{FIELD}`(§9)。`SourceSelector` 端口: `round_robin` / `least_inflight` 首发。**逻辑角色**: Video-Tree 式 SEARCH/JUDGE/VL/EVOLVE 多角色 = 命名的 client 配置组,`from_env()` 支持按角色前缀装配多个 client;禁止两个角色静默共享同一实例却在配置上看似独立(Video-Tree `evolve_llm = llm` 别名的教训——共享必须显式)。 + +### 7.8 遥测与成本 + +**必录字段**(继承三项目 15 字段规范): call_id、parent_call_id、session_id、model、provider、source_name、messages(JSON)、response、thinking、prompt_tokens、completion_tokens、usage_source、latency_ms、ttft_ms、max_inter_token_ms、cache_hit、error、**cost**。链路: `session_id`/`parent_call_id` 由调用方传入贯穿(agent step → LLM call)。 + +- 后端: `SQLiteRecorder`(默认;WAL + busy_timeout、`INSERT OR IGNORE` 幂等、`asyncio.to_thread` 桥接、初始化/写入失败全降级不冒泡)与 `PostgresRecorder`。 +- **单一 helper 铁律**: 遥测调用点收敛为一个内部函数/上下文管理器;Video-Tree 与 GovDoc 各有 4-5 处逐字复制的 `record_llm_call(15 个参数)` 是本条的直接教训。 +- 成本: `pricing.py` 维护 model → (input 单价, output 单价) 表,遥测时换算 `cost` 字段;查不到价格记 None 并 warning,**不阻塞调用**。 + +### 7.9 结构化输出策略 + +| 策略 | 机制 | 适用 | +|---|---|---| +| `JsonRepairStrategy` | prompt 约定 + ```json 围栏剥离 + json_repair + provider 变体归一化(如 DeepSeek 参数平铺) | 任意网关;三项目现状的收敛 | +| `NativeSchemaStrategy` | response_format json_schema / function calling | provider 注册表声明支持时 | + +逐调用可选;解析失败统一抛 `ResultInvalidError`(§6.3)。 + +### 7.10 OCR 端口族 + +| 端口 | 对应 MonkeyOCR 端点 | 协议 | 输出 | +|---|---|---|---| +| `OcrTextPort.recognize_text(image: bytes)` | `POST /ocr/text` | multipart 上传 → JSON `{content}` | `OcrTextResult`(多行纯文本) | +| `OcrLayoutPort.parse_layout(image: bytes)` | `POST /parse` | multipart → JSON(download_url) → GET ZIP → 解包 `*_middle.json` | `OcrLayoutResult`(elements 含 bbox + page_size) | + +设计要点:输入统一 `bytes`(路径读取/多帧批量拼接留业务侧);bbox 返回 OCR 原生页面坐标,几何映射(裁剪偏移/归一化/marker 推算)留业务侧;`None`/空表达"合法无内容",异常表达"调用失败";ZIP 内容不可解析抛 `ResultInvalidError`(坏图≠坏服务);OCR 走同一中间件栈(无 token 计费,Usage 填 0,elapsed 照记);多后端经 provider 注册表扩展(GLM 已在 CHSAnalyzer 白名单,输入形态为 URL,届时封装在其 invoker 内部,端口签名不变)。数值防御(bbox 有限性/顺序/退化校验)随协议解析下沉进库。 + +### 7.11 音频占位 + +`AudioPort` Protocol 占位,无实现、无 transport。真实需求出现时走 `designs/` 功能设计文档流程定协议形态。 + +--- + +## 8. 模块结构与依赖纪律 + +```text +src/polygateway/ +├── types.py # §5 核心类型(frozen dataclass) +├── errors.py # §6 错误四分类 +├── ports.py # 全部 Protocol(§4 各端口) +├── client.py # GatewayClient + from_env()/from_settings() 装配工厂 + gather_bounded +├── middleware/ # retry.py / ratelimit.py / breaker.py / cache.py / telemetry.py +├── transports/ # openai_compat.py / openai_sdk.py / monkey_ocr.py +├── providers.py # D11 provider 注册表 +├── sources.py # SourceConfig + 选源策略 +├── backends/ # memory/ 与 redis/(limiter、breaker、cache 状态实现) +├── telemetry/ # sqlite.py / postgres.py / pricing.py +├── structured/ # json_repair.py / native_schema.py +└── streaming.py # 三层活性看门狗(纯函数) +``` + +**依赖纪律**(import-linter 契约执法): `ports.py`/`types.py`/`errors.py` 为最内层,不 import 任何具体实现;`middleware/` 只依赖端口;`transports/`、`backends/`、`telemetry/`、`structured/` 只实现端口且互不依赖;`client.py` 是唯一的组装层。核心依赖仅 `httpx` + `pydantic`;`redis`/`aiosqlite`/`asyncpg`/`json_repair`/`openai` 全部 optional extras(`pip install polygateway[redis,telemetry-sqlite,...]`),import 失败时报清晰的"缺 extra"错误。 + +--- + +## 9. 配置面 + +- **载体**: `pydantic-settings` + `.env`(工程配置);缺失关键配置直接报错,严禁硬编码默认值兜底(三项目共同铁律)。 +- **多源命名**: `{SCOPE}__{PROVIDER}__{N}__{FIELD}`(如 `LLM__QWEN__1__API_KEY`、`OCR__MONKEY__1__BASE_URL`),聚合为 `list[SourceConfig]`;SCOPE 支持逻辑角色前缀(§7.7)。 +- **韧性参数键名**沿用三项目习惯(`LLM_TIMEOUT` / `LLM_MAX_RETRIES` / `LLM_RETRY_BASE_DELAY` / `LLM_RETRY_MAX_DELAY` / `LLM_CIRCUIT_BREAKER_THRESHOLD` / `LLM_CIRCUIT_BREAKER_COOLDOWN` / `LLM_TTFT_TIMEOUT` / `LLM_INTER_TOKEN_TIMEOUT`),降低三项目迁移改名成本。 +- **装配只有两条路**: `GatewayClient.from_env()`/`from_settings(settings)`(工厂,覆盖 90% 用户;补上三项目每次手写、GovDoc 缺失的"配置→client"一段)或构造函数全量依赖注入(测试/高级用户)。库内部任何组件**不得自读环境变量**(显式优于隐式)。 +- 后端选择即配置: 如 `PGW_LIMITER_BACKEND=memory|redis`、`PGW_TELEMETRY_BACKEND=sqlite|postgres`、`PGW_QUOTA_FULL=wait|fail_fast`(命名待 M1 设计文档定稿)。 + +--- + +## 10. 非功能性需求(强制覆盖,继承 Video-Tree CLAUDE.md §4.2.1 条款) + +| 维度 | 回答 | +|---|---| +| 持久化策略 | 遥测逐调用追加写(WAL);缓存写在响应成功后;崩溃最多丢当次调用的遥测记录 | +| 幂等性 | 遥测 `INSERT OR IGNORE`(call_id 主键);缓存写幂等(同 key 同值);限流 permit 带 TTL 租约,进程死亡后自动过期回收 | +| 断点续跑 | 库无长任务状态,天然无断点问题;响应缓存本身即业务侧重跑的加速器 | +| 原子性 | Redis 限流/熔断操作全部单条 Lua 原子执行;窗口 id 用 Redis 服务器时钟统一多进程口径 | +| 降级 | 缓存/遥测后端不可用 → 静默降级(warning);限流/熔断后端不可用 → **报错而非放行**(防击穿网关) | +| 取消 | `CancelledError` 全栈穿透;in-flight permit 与连接在 finally 释放(§6.4) | + +--- + +## 11. 三项目迁移路径(库的验收标准) + +> **验收定义**: 每个项目删除自己的治理实现文件,换成 `from polygateway import ...` + 配置,原测试全部通过。**凡替换不掉的能力,就是库的边界缺口**,回补后重验。这条标准同时是防"造没人用的空中楼阁"的机制:每个里程碑都有真实接入方。 + +### 11.1 GovDoc-SaaS(难度低,首个迁移) + +| 项目侧 | 处置 | +|---|---| +| `docagent-core/llm/client.py`、`breaker.py`、`redis_cache.py`、`streaming.py`、`telemetry_sqlite.py` | 删除,由库继任 | +| `protocols.py` 的 `LLMProvider.chat(messages, *, session_id, parent_call_id)` 签名 | 库保持兼容(或一行 shim) | +| 倒推的库需求 | `from_env` 工厂(GovDoc 装配层本就缺失,库直接补上)、Postgres 遥测、缓存 key namespace 含租户 | + +### 11.2 Video-Tree-TRM5(难度中) + +| 项目侧 | 处置 | +|---|---| +| `adapters/llm.py`、`breaker.py`、`streaming.py`、`redis_cache.py`、`telemetry.py` | 删除,由库继任 | +| `main.py:_build_adapters()` | 改为按角色调用 `from_env`(SEARCH/JUDGE/VL/EVOLVE;共享实例显式声明) | +| `adapters/vlm.py`(base64 编码与注入)、抽帧、OCR 文本拼接与注入前缀 | 留在项目(业务侧),组装好 content 数组后调库 | +| `adapters/ocr.py` 裸调轮询 | 删除,换 `OcrTextPort`(免费升级为多源+重试+熔断全治理) | +| 倒推的库需求 | 多逻辑角色、cache salt、多模态 content 摘要进 hash、`gather_bounded`、非流式快路径 | + +### 11.3 CHSAnalyzer(难度高,能力对标项) + +| 项目侧 | 处置 | +|---|---| +| `app/providers/governance.py`、`app/coordination/limiter.py` + `scripts.py`、`provider_gate.py` | 删除,由库继任(库必须先达到能力对等,这是 M2 的验收内容) | +| `app/providers/invokers.py` 的 VLM invoker / `MonkeyOcrParseInvoker` | 由库 transport / `OcrLayoutPort` 继任 | +| `app/providers/table_locator.py`(几何映射)、`marker_imaging.py`(拼图/增强)、`position_scheduler.py`(公平调度) | 留在项目(业务侧) | +| `core/eval/judge.py`(同步裸 SDK,反面教材) | 迁移到库,消灭无治理调用 | +| 倒推的库需求 | 多源多账号、Redis 六道闸+契约测试、错误四分类、OCR 端口族、背压 stall、`ResultInvalidError` 语义 | + +--- + +## 12. 里程碑 + +| 阶段 | 交付 | 可接入 | +|---|---|---| +| M1 核心 | types/errors/ports、OpenAICompat transport(含非流式)、看门狗、RetryMW、内存版限流/熔断、缓存(Redis+内存)、SQLite 遥测、结构化输出双策略、provider 注册表、from_env | GovDoc、Video-Tree | +| M2 分布式 | Redis 限流(六道闸+契约测试)/熔断后端、多源多账号+选源、背压 stall、Postgres 遥测、pricing 成本 | CHSAnalyzer(治理部分) | +| M3 OCR | OcrText/OcrLayout 端口 + MonkeyOCR transport,走同一治理栈 | CHSAnalyzer(全量)、Video-Tree(OCR 升级) | +| M4 迁移验证 | 三项目逐一按 §11 验收,缺口回补 | 全部 | + +每个里程碑进入实现前,按 SOP 在 `research-wiki/designs/` 写该里程碑的功能设计文档(受 400 行约束),与本文档冲突时先修订本文档。 + +--- + +## 13. 开放问题(待人类拍板) + +| # | 问题 | 建议 | +|---|---|---| +| Q1 | 打包与分发: 内网 pip index / git+ssh 依赖 / submodule? | git+ssh 起步,稳定后内网 index | +| Q2 | Python 最低版本 | 3.11(覆盖三项目: 3.11×2 + 3.13×1) | +| Q3 | Embedding 客户端是否纳入(GovDoc `retrieval/embedding.py` 与 Video-Tree `adapters/embedding.py` 各有一套独立重试实现,是第三处重复) | 建议 M2 纳入,复用同一治理栈 | +| Q4 | conda 环境名 | `PolyGateway` | +| Q5 | 本仓库工程脚手架(git init、`.claude/` skills、Makefile、pyproject、import-linter)何时落地 | 本文档终审通过后、M1 编码前一次落地 | diff --git a/src/polygateway/__init__.py b/src/polygateway/__init__.py new file mode 100644 index 0000000..97ea021 --- /dev/null +++ b/src/polygateway/__init__.py @@ -0,0 +1,7 @@ +"""PolyGateway：实验室统一的大语言模型（LLM/VLM/OCR）调度与中转库。 + +治理单位是一次模型调用：多源选择、限流、错误分类与重试、熔断、响应缓存、 +流式活性看门狗、遥测（含成本）。架构单一事实源见 research-wiki/ARCHITECTURE.md。 +""" + +__version__ = "0.1.0" diff --git a/tests/e2e/.gitkeep b/tests/e2e/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration/.gitkeep b/tests/integration/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/test_package.py b/tests/unit/test_package.py new file mode 100644 index 0000000..1d9c49e --- /dev/null +++ b/tests/unit/test_package.py @@ -0,0 +1,7 @@ +"""包基线冒烟测试：可导入、版本号存在。""" + +import polygateway + + +def test_package_importable_with_version() -> None: + assert polygateway.__version__ == "0.1.0"