chore: track claude skills, tools, templates, reference code and research-wiki
- Add all claude skills (brainstorming, commit, debugging, TDD, etc.) - Add claude hooks (pre-commit-guard, post-edit-quality) - Add research templates (experiment plan, research brief, etc.) - Add claude tools (arxiv/semantic_scholar/openalex fetch, wiki, exa) - Add TRM4 reference implementation as algorithm fidelity baseline - Add research-wiki content (plans, index, graph, query_pack) - Update .gitignore to exclude .graphify_version runtime state
This commit is contained in:
@@ -0,0 +1,185 @@
|
||||
"""arXiv 搜索与 PDF 下载的独立命令行工具。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
import xml.etree.ElementTree as ET
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
|
||||
ATOM_NS = "http://www.w3.org/2005/Atom"
|
||||
ARXIV_API_URL = (
|
||||
"http://export.arxiv.org/api/query?search_query=all:{query}&max_results={max}"
|
||||
)
|
||||
ARXIV_PDF_URL = "https://arxiv.org/pdf/{arxiv_id}.pdf"
|
||||
USER_AGENT = "arxiv_fetch.py/1.0"
|
||||
MIN_PDF_SIZE = 10 * 1024
|
||||
DOWNLOAD_DELAY_SECONDS = 1.0
|
||||
|
||||
|
||||
def _clean_text(value: str) -> str:
|
||||
"""压缩空白并清理文本内容。"""
|
||||
return " ".join(value.split())
|
||||
|
||||
|
||||
def _normalize_arxiv_id(raw_id: str) -> str:
|
||||
"""将 arXiv 标识归一化为不带版本号的论文 ID。"""
|
||||
value = raw_id.strip()
|
||||
if value.startswith("http://") or value.startswith("https://"):
|
||||
value = value.rsplit("/", 1)[-1]
|
||||
if value.startswith("abs/") or value.startswith("pdf/"):
|
||||
value = value.split("/", 1)[1]
|
||||
if value.endswith(".pdf"):
|
||||
value = value[:-4]
|
||||
if value.startswith("arXiv:") or value.startswith("arxiv:"):
|
||||
value = value.split(":", 1)[1]
|
||||
if "v" in value:
|
||||
head, tail = value.rsplit("v", 1)
|
||||
if tail.isdigit():
|
||||
value = head
|
||||
return value
|
||||
|
||||
|
||||
def _request_text(url: str) -> bytes:
|
||||
"""发起 HTTP 请求并返回原始字节内容。"""
|
||||
request = Request(url, headers={"User-Agent": USER_AGENT})
|
||||
with urlopen(request) as response:
|
||||
return response.read()
|
||||
|
||||
|
||||
def _parse_arxiv_feed(xml_bytes: bytes) -> list[dict[str, Any]]:
|
||||
"""解析 arXiv Atom Feed 并提取论文条目。"""
|
||||
root = ET.fromstring(xml_bytes)
|
||||
namespace = {"atom": ATOM_NS}
|
||||
results: list[dict[str, Any]] = []
|
||||
|
||||
for entry in root.findall("atom:entry", namespace):
|
||||
title = _clean_text(
|
||||
entry.findtext("atom:title", default="", namespaces=namespace)
|
||||
)
|
||||
abstract = _clean_text(
|
||||
entry.findtext("atom:summary", default="", namespaces=namespace)
|
||||
)
|
||||
published = _clean_text(
|
||||
entry.findtext("atom:published", default="", namespaces=namespace)
|
||||
)
|
||||
if "T" in published:
|
||||
published = published.split("T", 1)[0]
|
||||
|
||||
authors: list[str] = []
|
||||
for author in entry.findall("atom:author", namespace):
|
||||
name = _clean_text(
|
||||
author.findtext("atom:name", default="", namespaces=namespace)
|
||||
)
|
||||
if name:
|
||||
authors.append(name)
|
||||
|
||||
categories: list[str] = []
|
||||
for category in entry.findall("atom:category", namespace):
|
||||
term = category.get("term", "").strip()
|
||||
if term and term not in categories:
|
||||
categories.append(term)
|
||||
|
||||
raw_id = entry.findtext("atom:id", default="", namespaces=namespace)
|
||||
arxiv_id = _normalize_arxiv_id(raw_id.rsplit("/", 1)[-1] if raw_id else "")
|
||||
|
||||
results.append(
|
||||
{
|
||||
"title": title,
|
||||
"authors": authors,
|
||||
"arxiv_id": arxiv_id,
|
||||
"abstract": abstract,
|
||||
"categories": categories,
|
||||
"published": published,
|
||||
}
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def search_arxiv(query: str, max_results: int) -> list[dict[str, Any]]:
|
||||
"""搜索 arXiv 并返回结构化论文结果。"""
|
||||
encoded_query = quote(query, safe="")
|
||||
url = ARXIV_API_URL.format(query=encoded_query, max=max_results)
|
||||
feed_bytes = _request_text(url)
|
||||
return _parse_arxiv_feed(feed_bytes)
|
||||
|
||||
|
||||
def download_arxiv_pdf(arxiv_id: str, target_dir: Path) -> Path:
|
||||
"""下载指定 arXiv 论文的 PDF 并校验文件大小。"""
|
||||
normalized_id = _normalize_arxiv_id(arxiv_id)
|
||||
target_dir.mkdir(parents=True, exist_ok=True)
|
||||
output_path = target_dir / f"{normalized_id}.pdf"
|
||||
pdf_url = ARXIV_PDF_URL.format(arxiv_id=normalized_id)
|
||||
|
||||
request = Request(pdf_url, headers={"User-Agent": USER_AGENT})
|
||||
time.sleep(DOWNLOAD_DELAY_SECONDS)
|
||||
try:
|
||||
with urlopen(request) as response, output_path.open("wb") as handle:
|
||||
while True:
|
||||
chunk = response.read(8192)
|
||||
if not chunk:
|
||||
break
|
||||
handle.write(chunk)
|
||||
except Exception:
|
||||
if output_path.exists():
|
||||
output_path.unlink()
|
||||
raise
|
||||
|
||||
if output_path.stat().st_size <= MIN_PDF_SIZE:
|
||||
output_path.unlink(missing_ok=True)
|
||||
raise ValueError(f"下载文件过小，疑似失败: {output_path}")
|
||||
|
||||
return output_path
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
"""构建命令行参数解析器。"""
|
||||
parser = argparse.ArgumentParser(description="arXiv 搜索与 PDF 下载工具")
|
||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
search_parser = subparsers.add_parser("search", help="搜索 arXiv 论文")
|
||||
search_parser.add_argument("query", help="搜索关键词")
|
||||
search_parser.add_argument("--max", type=int, default=10, help="最大返回数量")
|
||||
|
||||
download_parser = subparsers.add_parser("download", help="下载 arXiv PDF")
|
||||
download_parser.add_argument("arxiv_id", help="arXiv 论文 ID")
|
||||
download_parser.add_argument(
|
||||
"--dir", dest="target_dir", default=".", help="保存目录"
|
||||
)
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
"""CLI 入口。"""
|
||||
parser = build_parser()
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
try:
|
||||
if args.command == "search":
|
||||
results = search_arxiv(args.query, args.max)
|
||||
json.dump(results, sys.stdout, ensure_ascii=False, indent=2)
|
||||
sys.stdout.write("\n")
|
||||
return 0
|
||||
|
||||
if args.command == "download":
|
||||
output_path = download_arxiv_pdf(args.arxiv_id, Path(args.target_dir))
|
||||
sys.stdout.write(f"{output_path}\n")
|
||||
return 0
|
||||
except Exception as exc:
|
||||
print(f"错误: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
parser.error(f"未知命令: {args.command}")
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,52 @@
|
||||
"""DeepXiv CLI 的轻量适配器。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import cast
|
||||
|
||||
if shutil.which("deepxiv") is None:
|
||||
print("警告: 未找到 deepxiv CLI，已跳过 DeepXiv 适配器。", file=sys.stderr)
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
def _run_deepxiv(arguments: list[str]) -> int:
|
||||
"""调用底层 deepxiv 命令并透传输出。"""
|
||||
completed = subprocess.run(["deepxiv", *arguments], check=False)
|
||||
return completed.returncode
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
"""构建命令行参数解析器。"""
|
||||
parser = argparse.ArgumentParser(description="DeepXiv 命令适配器")
|
||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
for command_name in ("search", "paper-brief", "paper-head", "paper-section"):
|
||||
subparser = subparsers.add_parser(
|
||||
command_name, help=f"转发 deepxiv {command_name} 命令"
|
||||
)
|
||||
subparser.add_argument(
|
||||
"args", nargs=argparse.REMAINDER, help="透传给 deepxiv 的参数"
|
||||
)
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
"""CLI 入口。"""
|
||||
parser = build_parser()
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
try:
|
||||
command_args = [args.command, *cast(list[str], args.args)]
|
||||
return _run_deepxiv(command_args)
|
||||
except OSError as exc:
|
||||
print(f"错误: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,144 @@
|
||||
"""Exa 搜索命令行工具。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from dataclasses import asdict, is_dataclass
|
||||
from typing import Any
|
||||
|
||||
try:
|
||||
import exa
|
||||
except ImportError:
|
||||
print("警告: 未安装 exa-py SDK，已跳过 Exa CLI。", file=sys.stderr)
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
def _to_jsonable(value: Any) -> Any:
|
||||
"""将任意 SDK 返回值递归转换为可序列化对象。"""
|
||||
if value is None or isinstance(value, (str, int, float, bool)):
|
||||
return value
|
||||
if isinstance(value, bytes):
|
||||
return value.decode("utf-8", errors="replace")
|
||||
if isinstance(value, dict):
|
||||
return {str(key): _to_jsonable(item) for key, item in value.items()}
|
||||
if isinstance(value, (list, tuple, set)):
|
||||
return [_to_jsonable(item) for item in value]
|
||||
if is_dataclass(value):
|
||||
return _to_jsonable(asdict(value))
|
||||
model_dump = getattr(value, "model_dump", None)
|
||||
if callable(model_dump):
|
||||
return _to_jsonable(model_dump())
|
||||
if hasattr(value, "__dict__"):
|
||||
data = {
|
||||
key: _to_jsonable(item)
|
||||
for key, item in vars(value).items()
|
||||
if not key.startswith("_")
|
||||
}
|
||||
if data:
|
||||
return data
|
||||
return str(value)
|
||||
|
||||
|
||||
def _build_contents_option(content_mode: str) -> dict[str, Any]:
|
||||
"""根据命令行参数构造 Exa contents 配置。"""
|
||||
if content_mode == "text":
|
||||
return {"text": True}
|
||||
return {"highlights": True}
|
||||
|
||||
|
||||
def _extract_results(response: Any) -> list[dict[str, Any]]:
|
||||
"""从 Exa 响应中提取结果列表并标准化为 JSON 对象。"""
|
||||
raw_results: Any
|
||||
if isinstance(response, dict):
|
||||
raw_results = response.get("results", [])
|
||||
else:
|
||||
raw_results = getattr(response, "results", [])
|
||||
|
||||
if not isinstance(raw_results, list):
|
||||
return []
|
||||
|
||||
normalized_results: list[dict[str, Any]] = []
|
||||
for item in raw_results:
|
||||
normalized_item = _to_jsonable(item)
|
||||
if isinstance(normalized_item, dict):
|
||||
normalized_results.append(normalized_item)
|
||||
else:
|
||||
normalized_results.append({"value": normalized_item})
|
||||
return normalized_results
|
||||
|
||||
|
||||
def search_exa(
|
||||
query: str,
|
||||
max_results: int,
|
||||
category: str | None,
|
||||
content_mode: str,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""执行 Exa 搜索并返回结果列表。"""
|
||||
api_key = os.environ.get("EXA_API_KEY", "").strip()
|
||||
if not api_key:
|
||||
print("警告: 缺少 EXA_API_KEY 环境变量，已跳过 Exa CLI。", file=sys.stderr)
|
||||
sys.exit(0)
|
||||
|
||||
client = exa.Exa(api_key=api_key)
|
||||
search_kwargs: dict[str, Any] = {
|
||||
"num_results": max_results,
|
||||
"contents": _build_contents_option(content_mode),
|
||||
}
|
||||
if category:
|
||||
search_kwargs["category"] = category
|
||||
|
||||
response = client.search(query, **search_kwargs)
|
||||
return _extract_results(response)
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
"""构建命令行参数解析器。"""
|
||||
parser = argparse.ArgumentParser(description="Exa 搜索工具")
|
||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
search_parser = subparsers.add_parser("search", help="搜索 Exa 索引")
|
||||
search_parser.add_argument("query", help="搜索关键词")
|
||||
search_parser.add_argument("--max", dest="max_results", type=int, default=10)
|
||||
search_parser.add_argument(
|
||||
"--category",
|
||||
default=None,
|
||||
help="数据类别，例如 research paper、news、pdf 等",
|
||||
)
|
||||
search_parser.add_argument(
|
||||
"--content",
|
||||
choices=("highlights", "text"),
|
||||
default="highlights",
|
||||
help="返回的内容类型",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
"""CLI 入口。"""
|
||||
parser = build_parser()
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
try:
|
||||
if args.command == "search":
|
||||
results = search_exa(
|
||||
query=args.query,
|
||||
max_results=args.max_results,
|
||||
category=args.category,
|
||||
content_mode=args.content,
|
||||
)
|
||||
json.dump(results, sys.stdout, ensure_ascii=False, indent=2)
|
||||
sys.stdout.write("\n")
|
||||
return 0
|
||||
except Exception as exc:
|
||||
print(f"错误: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
parser.error(f"未知命令: {args.command}")
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,190 @@
|
||||
"""OpenAlex 搜索命令行工具。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
try:
|
||||
import requests
|
||||
except ImportError:
|
||||
print("警告: 未安装 requests，已跳过 OpenAlex CLI。", file=sys.stderr)
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
API_URL = "https://api.openalex.org/works"
|
||||
REQUEST_TIMEOUT_SECONDS = 30.0
|
||||
USER_AGENT = "openalex_fetch.py/1.0"
|
||||
|
||||
|
||||
def _clean_text(value: Any) -> str:
|
||||
"""压缩空白并清理文本值。"""
|
||||
return " ".join(str(value).split())
|
||||
|
||||
|
||||
def _normalize_year_filter(raw_year: str) -> str:
|
||||
"""将命令行年份参数转换为 OpenAlex filter 语法。"""
|
||||
value = raw_year.strip()
|
||||
if not value:
|
||||
raise ValueError("年份参数不能为空")
|
||||
|
||||
if value.isdigit() and len(value) == 4:
|
||||
return f"publication_year:{value}"
|
||||
|
||||
if value.endswith("-") and value[:-1].strip().isdigit():
|
||||
start_year = value[:-1].strip()
|
||||
return f"from_publication_date:{start_year}-01-01"
|
||||
|
||||
if value.startswith("-") and value[1:].strip().isdigit():
|
||||
end_year = value[1:].strip()
|
||||
return f"to_publication_date:{end_year}-12-31"
|
||||
|
||||
if "-" in value:
|
||||
start_year, end_year = [part.strip() for part in value.split("-", 1)]
|
||||
if start_year.isdigit() and end_year.isdigit():
|
||||
return f"publication_year:{start_year}-{end_year}"
|
||||
|
||||
raise ValueError(f"无法识别的年份参数: {raw_year}")
|
||||
|
||||
|
||||
def _normalize_sort_value(sort_value: str) -> str:
|
||||
"""将用户输入的排序参数转换为 OpenAlex sort 语法。"""
|
||||
value = sort_value.strip()
|
||||
if not value:
|
||||
raise ValueError("排序参数不能为空")
|
||||
if value in {"relevance", "relevance_score"}:
|
||||
return "relevance_score:desc"
|
||||
return value
|
||||
|
||||
|
||||
def _normalize_work(work: dict[str, Any]) -> dict[str, Any]:
|
||||
"""将 OpenAlex work 记录压缩成稳定输出结构。"""
|
||||
authorships = work.get("authorships")
|
||||
authors: list[str] = []
|
||||
institutions: list[str] = []
|
||||
if isinstance(authorships, list):
|
||||
for authorship in authorships:
|
||||
if not isinstance(authorship, dict):
|
||||
continue
|
||||
author = authorship.get("author")
|
||||
if isinstance(author, dict):
|
||||
display_name = author.get("display_name")
|
||||
if display_name:
|
||||
cleaned_name = _clean_text(display_name)
|
||||
if cleaned_name not in authors:
|
||||
authors.append(cleaned_name)
|
||||
author_institutions = authorship.get("institutions")
|
||||
if isinstance(author_institutions, list):
|
||||
for institution in author_institutions:
|
||||
if not isinstance(institution, dict):
|
||||
continue
|
||||
display_name = institution.get("display_name")
|
||||
if display_name:
|
||||
cleaned_name = _clean_text(display_name)
|
||||
if cleaned_name not in institutions:
|
||||
institutions.append(cleaned_name)
|
||||
|
||||
return {
|
||||
"title": _clean_text(work.get("display_name", "")),
|
||||
"authors": authors,
|
||||
"year": work.get("publication_year"),
|
||||
"cited_by_count": work.get("cited_by_count"),
|
||||
"institutions": institutions,
|
||||
"doi": work.get("doi"),
|
||||
}
|
||||
|
||||
|
||||
def search_openalex(
|
||||
query: str,
|
||||
max_results: int,
|
||||
year: str | None,
|
||||
work_type: str | None,
|
||||
sort: str,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""搜索 OpenAlex 并返回标准化论文结果。"""
|
||||
params: dict[str, Any] = {
|
||||
"search": query,
|
||||
"per-page": max_results,
|
||||
"sort": _normalize_sort_value(sort),
|
||||
}
|
||||
|
||||
filters: list[str] = []
|
||||
if year:
|
||||
filters.append(_normalize_year_filter(year))
|
||||
if work_type:
|
||||
filters.append(f"type:{work_type.strip()}")
|
||||
if filters:
|
||||
params["filter"] = ",".join(filters)
|
||||
|
||||
response = requests.get(
|
||||
API_URL,
|
||||
params=params,
|
||||
headers={"User-Agent": USER_AGENT},
|
||||
timeout=REQUEST_TIMEOUT_SECONDS,
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
payload = response.json()
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError("OpenAlex 返回了非对象类型的 JSON")
|
||||
|
||||
works = payload.get("results", [])
|
||||
if not isinstance(works, list):
|
||||
raise ValueError("OpenAlex 响应缺少 results 列表")
|
||||
|
||||
normalized_results: list[dict[str, Any]] = []
|
||||
for work in works:
|
||||
if isinstance(work, dict):
|
||||
normalized_results.append(_normalize_work(work))
|
||||
return normalized_results
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
"""构建命令行参数解析器。"""
|
||||
parser = argparse.ArgumentParser(description="OpenAlex 搜索工具")
|
||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
search_parser = subparsers.add_parser("search", help="搜索 OpenAlex works")
|
||||
search_parser.add_argument("query", help="搜索关键词")
|
||||
search_parser.add_argument("--max", dest="max_results", type=int, default=10)
|
||||
search_parser.add_argument("--year", default=None, help="年份或年份范围")
|
||||
search_parser.add_argument(
|
||||
"--type", dest="work_type", default=None, help="作品类型"
|
||||
)
|
||||
search_parser.add_argument(
|
||||
"--sort",
|
||||
default="relevance",
|
||||
help="排序字段，relevance 会映射为 relevance_score:desc",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
"""CLI 入口。"""
|
||||
parser = build_parser()
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
try:
|
||||
if args.command == "search":
|
||||
results = search_openalex(
|
||||
query=args.query,
|
||||
max_results=args.max_results,
|
||||
year=args.year,
|
||||
work_type=args.work_type,
|
||||
sort=args.sort,
|
||||
)
|
||||
json.dump(results, sys.stdout, ensure_ascii=False, indent=2)
|
||||
sys.stdout.write("\n")
|
||||
return 0
|
||||
except (requests.RequestException, ValueError) as exc:
|
||||
print(f"错误: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
parser.error(f"未知命令: {args.command}")
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,681 @@
|
||||
"""研究 Wiki 的初始化与日志工具。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import unicodedata
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.request import urlopen
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
ENTITY_TYPES = {
|
||||
"paper": "papers",
|
||||
"idea": "ideas",
|
||||
"experiment": "experiments",
|
||||
"claim": "claims",
|
||||
"gap": "gaps",
|
||||
"design": "designs",
|
||||
"finding": "findings",
|
||||
"adr": "adrs",
|
||||
"plan": "plans",
|
||||
"review": "reviews",
|
||||
"schema": "schemas",
|
||||
"metric": "metrics",
|
||||
}
|
||||
|
||||
EDGE_TYPES = [
|
||||
"extends",
|
||||
"contradicts",
|
||||
"supersedes",
|
||||
"addresses_gap",
|
||||
"inspired_by",
|
||||
"tested_by",
|
||||
"supports",
|
||||
"invalidates",
|
||||
"implements",
|
||||
"reveals",
|
||||
"informs",
|
||||
"refines",
|
||||
"depends_on",
|
||||
"measures",
|
||||
"evaluates",
|
||||
]
|
||||
|
||||
QUERY_PACK_BUDGET = 8000
|
||||
|
||||
|
||||
def slugify(text: str) -> str:
|
||||
"""将文本转换为 URL 安全的 slug。"""
|
||||
normalized = unicodedata.normalize("NFKD", text)
|
||||
normalized = re.sub(r"[^\w\s-]", "", normalized.lower())
|
||||
return re.sub(r"[-\s]+", "_", normalized).strip("_")[:60]
|
||||
|
||||
|
||||
def _write_if_missing(path: Path, content: str) -> None:
|
||||
"""仅在文件不存在时写入内容。
|
||||
|
||||
参数:
|
||||
path: 目标文件路径。
|
||||
content: 需要写入的文本内容。
|
||||
"""
|
||||
if path.exists():
|
||||
return
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(content, encoding="utf-8")
|
||||
|
||||
|
||||
def append_log(wiki_dir: str, message: str) -> None:
|
||||
"""向 Wiki 变更日志追加一条记录。
|
||||
|
||||
参数:
|
||||
wiki_dir: Wiki 根目录。
|
||||
message: 需要记录的消息。
|
||||
"""
|
||||
log_path = Path(wiki_dir) / "log.md"
|
||||
timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
|
||||
with log_path.open("a", encoding="utf-8") as handle:
|
||||
handle.write(f"- [{timestamp}] {message}\n")
|
||||
|
||||
|
||||
def _render_frontmatter(data: dict[str, Any]) -> str:
|
||||
"""将字典渲染为 YAML frontmatter。"""
|
||||
lines = ["---"]
|
||||
for key, value in data.items():
|
||||
if isinstance(value, list):
|
||||
rendered_value = json.dumps(value, ensure_ascii=False)
|
||||
elif isinstance(value, str) and " " in value:
|
||||
rendered_value = f'"{value}"'
|
||||
else:
|
||||
rendered_value = str(value)
|
||||
lines.append(f"{key}: {rendered_value}")
|
||||
lines.append("---")
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def _read_frontmatter(filepath: Path) -> dict[str, Any]:
|
||||
"""从 markdown 文件读取简易 frontmatter。"""
|
||||
text = filepath.read_text(encoding="utf-8")
|
||||
if not text.startswith("---\n"):
|
||||
return {}
|
||||
|
||||
lines = text.splitlines()
|
||||
result: dict[str, Any] = {}
|
||||
for line in lines[1:]:
|
||||
if line == "---":
|
||||
break
|
||||
if ":" not in line:
|
||||
continue
|
||||
key, _, raw_value = line.partition(":")
|
||||
value = raw_value.strip()
|
||||
if value.startswith("[") and value.endswith("]"):
|
||||
try:
|
||||
result[key.strip()] = json.loads(value)
|
||||
continue
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
if len(value) >= 2 and (
|
||||
(value.startswith('"') and value.endswith('"'))
|
||||
or (value.startswith("'") and value.endswith("'"))
|
||||
):
|
||||
value = value[1:-1]
|
||||
result[key.strip()] = value
|
||||
return result
|
||||
|
||||
|
||||
def _add_node_to_graph(
|
||||
wiki_dir: str, node_id: str, label: str, entity_type: str
|
||||
) -> None:
|
||||
"""向关系图中添加节点，若已存在则跳过。"""
|
||||
graph_path = Path(wiki_dir) / "graph" / "edges.json"
|
||||
graph = json.loads(graph_path.read_text(encoding="utf-8"))
|
||||
nodes = graph.setdefault("nodes", [])
|
||||
if not any(node.get("id") == node_id for node in nodes):
|
||||
nodes.append({"id": node_id, "label": label, "type": entity_type})
|
||||
graph_path.write_text(
|
||||
json.dumps(graph, ensure_ascii=False, indent=2), encoding="utf-8"
|
||||
)
|
||||
|
||||
|
||||
def add_edge(
|
||||
wiki_dir: str,
|
||||
from_id: str,
|
||||
to_id: str,
|
||||
edge_type: str,
|
||||
evidence: str = "",
|
||||
) -> None:
|
||||
"""向关系图中添加一条边。"""
|
||||
if edge_type not in EDGE_TYPES:
|
||||
raise ValueError(f"未知边类型: {edge_type}，合法值: {EDGE_TYPES}")
|
||||
|
||||
graph_path = Path(wiki_dir) / "graph" / "edges.json"
|
||||
graph = json.loads(graph_path.read_text(encoding="utf-8"))
|
||||
links = graph.setdefault("links", [])
|
||||
|
||||
for link in links:
|
||||
if (
|
||||
link.get("source") == from_id
|
||||
and link.get("target") == to_id
|
||||
and link.get("relation") == edge_type
|
||||
):
|
||||
return
|
||||
|
||||
links.append(
|
||||
{
|
||||
"source": from_id,
|
||||
"target": to_id,
|
||||
"relation": edge_type,
|
||||
"evidence": evidence,
|
||||
"added": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
)
|
||||
graph_path.write_text(
|
||||
json.dumps(graph, ensure_ascii=False, indent=2), encoding="utf-8"
|
||||
)
|
||||
append_log(wiki_dir, f"新增边: {from_id} --{edge_type}--> {to_id}")
|
||||
|
||||
|
||||
def add_entity(
|
||||
wiki_dir: str,
|
||||
entity_type: str,
|
||||
entity_id: str,
|
||||
title: str,
|
||||
extra_frontmatter: dict[str, Any] | None = None,
|
||||
) -> Path:
|
||||
"""创建一个实体页面并同步到关系图。"""
|
||||
if entity_type not in ENTITY_TYPES:
|
||||
raise ValueError(f"未知实体类型: {entity_type}，合法值: {list(ENTITY_TYPES)}")
|
||||
|
||||
root = Path(wiki_dir)
|
||||
entity_subdir = root / ENTITY_TYPES[entity_type]
|
||||
node_id = f"{entity_type}:{entity_id}"
|
||||
|
||||
for existing_path in entity_subdir.glob("*.md"):
|
||||
frontmatter = _read_frontmatter(existing_path)
|
||||
if frontmatter.get("node_id") == node_id:
|
||||
return existing_path
|
||||
|
||||
filepath = entity_subdir / f"{entity_id}.md"
|
||||
frontmatter: dict[str, Any] = {
|
||||
"type": entity_type,
|
||||
"node_id": node_id,
|
||||
"title": title,
|
||||
"date": datetime.now(timezone.utc).strftime("%Y-%m-%d"),
|
||||
}
|
||||
if extra_frontmatter:
|
||||
frontmatter.update(extra_frontmatter)
|
||||
|
||||
content = _render_frontmatter(frontmatter) + f"\n# {title}\n\n"
|
||||
filepath.write_text(content, encoding="utf-8")
|
||||
|
||||
_add_node_to_graph(wiki_dir, node_id, title, entity_type)
|
||||
append_log(wiki_dir, f"新增 {entity_type}: {title} ({node_id})")
|
||||
return filepath
|
||||
|
||||
|
||||
def _fetch_arxiv_metadata(arxiv_id: str) -> dict[str, Any] | None:
|
||||
"""通过 arXiv Atom API 拉取论文元数据。"""
|
||||
clean_id = (
|
||||
arxiv_id.removeprefix("arxiv:")
|
||||
.removeprefix("arXiv:")
|
||||
.removeprefix("ARXIV:")
|
||||
.strip()
|
||||
)
|
||||
url = f"http://export.arxiv.org/api/query?id_list={clean_id}"
|
||||
|
||||
try:
|
||||
with urlopen(url) as response:
|
||||
root = ET.fromstring(response.read())
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
namespace = {"atom": "http://www.w3.org/2005/Atom"}
|
||||
entry = root.find("atom:entry", namespace)
|
||||
if entry is None:
|
||||
return None
|
||||
|
||||
def _text(path: str) -> str:
|
||||
value = entry.findtext(path, default="", namespaces=namespace)
|
||||
return " ".join(value.split())
|
||||
|
||||
authors = [
|
||||
" ".join(author.findtext("atom:name", default="", namespaces=namespace).split())
|
||||
for author in entry.findall("atom:author", namespace)
|
||||
]
|
||||
authors = [author for author in authors if author]
|
||||
published = _text("atom:published")
|
||||
year = published[:4] if len(published) >= 4 else ""
|
||||
|
||||
return {
|
||||
"title": _text("atom:title"),
|
||||
"authors": authors,
|
||||
"year": year,
|
||||
"abstract": _text("atom:summary"),
|
||||
"arxiv_id": clean_id,
|
||||
}
|
||||
|
||||
|
||||
def _last_name(full_name: str) -> str:
|
||||
"""提取人名最后一个词作为姓氏。"""
|
||||
parts = full_name.strip().split()
|
||||
return parts[-1].lower() if parts else "unknown"
|
||||
|
||||
|
||||
def _markdown_body(text: str) -> str:
|
||||
"""提取 markdown 中 frontmatter 之后的正文。"""
|
||||
if not text.startswith("---\n"):
|
||||
return text
|
||||
|
||||
lines = text.splitlines()
|
||||
for index, line in enumerate(lines[1:], start=1):
|
||||
if line == "---":
|
||||
return "\n".join(lines[index + 1 :])
|
||||
return ""
|
||||
|
||||
|
||||
def _iter_entity_pages(wiki_dir: str) -> list[tuple[str, Path, dict[str, Any], str]]:
|
||||
"""收集 Wiki 中所有实体页面。"""
|
||||
root = Path(wiki_dir)
|
||||
pages: list[tuple[str, Path, dict[str, Any], str]] = []
|
||||
|
||||
for entity_type, subdir_name in ENTITY_TYPES.items():
|
||||
subdir = root / subdir_name
|
||||
for path in sorted(subdir.glob("*.md")):
|
||||
frontmatter = _read_frontmatter(path)
|
||||
body = _markdown_body(path.read_text(encoding="utf-8"))
|
||||
resolved_type = (
|
||||
str(frontmatter.get("type", entity_type)).strip() or entity_type
|
||||
)
|
||||
pages.append((resolved_type, path, frontmatter, body))
|
||||
|
||||
return pages
|
||||
|
||||
|
||||
def rebuild_index(wiki_dir: str) -> None:
|
||||
"""按实体类型重建索引页。"""
|
||||
root = Path(wiki_dir)
|
||||
timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
|
||||
pages = _iter_entity_pages(wiki_dir)
|
||||
|
||||
grouped: dict[str, list[tuple[Path, dict[str, Any]]]] = {}
|
||||
for entity_type, path, frontmatter, _ in pages:
|
||||
grouped.setdefault(entity_type, []).append((path, frontmatter))
|
||||
|
||||
lines = [
|
||||
"# Research Wiki 索引",
|
||||
"",
|
||||
f"> 自动生成，更新时间：{timestamp}",
|
||||
"",
|
||||
]
|
||||
|
||||
for entity_type in ENTITY_TYPES:
|
||||
entries = grouped.get(entity_type, [])
|
||||
if not entries:
|
||||
continue
|
||||
entries = sorted(
|
||||
entries,
|
||||
key=lambda item: (
|
||||
str(item[1].get("title", item[0].stem)).casefold(),
|
||||
item[0].name,
|
||||
),
|
||||
)
|
||||
lines.append(f"## {entity_type} ({len(entries)})")
|
||||
for path, frontmatter in entries:
|
||||
title = str(frontmatter.get("title", path.stem)).strip() or path.stem
|
||||
node_id = str(
|
||||
frontmatter.get("node_id", f"{entity_type}:{path.stem}")
|
||||
).strip()
|
||||
relative_path = path.relative_to(root).as_posix()
|
||||
lines.append(f"- [{title}]({relative_path}) `{node_id}`")
|
||||
lines.append("")
|
||||
|
||||
content = "\n".join(lines).rstrip() + "\n"
|
||||
(root / "index.md").write_text(content, encoding="utf-8")
|
||||
append_log(wiki_dir, f"重建索引: {len(pages)} 篇页面")
|
||||
|
||||
|
||||
def rebuild_query_pack(wiki_dir: str) -> None:
|
||||
"""基于 frontmatter 重新生成 Query Pack。"""
|
||||
pages = _iter_entity_pages(wiki_dir)
|
||||
|
||||
failed_ideas: list[tuple[str, str]] = []
|
||||
open_gaps: list[str] = []
|
||||
core_papers: list[tuple[str, str, str]] = []
|
||||
|
||||
for entity_type, path, frontmatter, _ in pages:
|
||||
title = str(frontmatter.get("title", path.stem)).strip() or path.stem
|
||||
if entity_type == "idea":
|
||||
outcome = str(frontmatter.get("outcome", "")).strip().lower()
|
||||
if outcome in {"negative", "failed"}:
|
||||
failure_reason = str(frontmatter.get("failure_reason", "")).strip()
|
||||
failed_ideas.append((title, failure_reason or "未填写"))
|
||||
if entity_type == "gap":
|
||||
status = str(frontmatter.get("status", "open")).strip().lower()
|
||||
if not status or status == "open":
|
||||
open_gaps.append(title)
|
||||
if entity_type == "paper":
|
||||
year = str(frontmatter.get("year", "")).strip()
|
||||
venue = str(frontmatter.get("venue", "")).strip()
|
||||
core_papers.append(
|
||||
(
|
||||
title,
|
||||
year or "未知年份",
|
||||
venue or "未知 venue",
|
||||
)
|
||||
)
|
||||
|
||||
failed_ideas.sort(key=lambda item: item[0].casefold())
|
||||
open_gaps.sort(key=str.casefold)
|
||||
core_papers.sort(key=lambda item: (item[1], item[0].casefold(), item[2].casefold()))
|
||||
|
||||
sections: list[tuple[str, list[str]]] = []
|
||||
if failed_ideas:
|
||||
sections.append(
|
||||
(
|
||||
"## 失败 Idea（禁区）",
|
||||
[f"- ❌ {title}: {reason}" for title, reason in failed_ideas[:10]],
|
||||
)
|
||||
)
|
||||
if open_gaps:
|
||||
sections.append(
|
||||
(
|
||||
"## 未解决空白",
|
||||
[f"- {title}" for title in open_gaps[:5]],
|
||||
)
|
||||
)
|
||||
if core_papers:
|
||||
sections.append(
|
||||
(
|
||||
"## 核心论文",
|
||||
[
|
||||
f"- [{title}] ({year}) {venue}"
|
||||
for title, year, venue in core_papers[:12]
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
content_parts = [
|
||||
"# Query Pack",
|
||||
"",
|
||||
"> 自动生成，请勿手动编辑。",
|
||||
"",
|
||||
"",
|
||||
]
|
||||
current_content = "\n".join(content_parts)
|
||||
|
||||
for section_title, section_lines in sections:
|
||||
section_text = "\n".join([section_title, *section_lines, "", ""])
|
||||
candidate = current_content + section_text
|
||||
if len(candidate) > QUERY_PACK_BUDGET:
|
||||
break
|
||||
current_content = candidate
|
||||
|
||||
content = current_content.rstrip() + "\n"
|
||||
(Path(wiki_dir) / "query_pack.md").write_text(content, encoding="utf-8")
|
||||
append_log(wiki_dir, f"重建 Query Pack: {len(content)} 字符")
|
||||
|
||||
|
||||
def get_stats(wiki_dir: str) -> str:
|
||||
"""统计 Wiki 规模与图谱信息。"""
|
||||
root = Path(wiki_dir)
|
||||
lines: list[str] = []
|
||||
|
||||
for entity_type, subdir_name in ENTITY_TYPES.items():
|
||||
count = len(list((root / subdir_name).glob("*.md")))
|
||||
lines.append(f"{entity_type}: {count}")
|
||||
|
||||
graph_path = root / "graph" / "edges.json"
|
||||
if graph_path.exists():
|
||||
graph = json.loads(graph_path.read_text(encoding="utf-8"))
|
||||
else:
|
||||
graph = {"nodes": [], "links": []}
|
||||
|
||||
lines.append(f"nodes: {len(graph.get('nodes', []))}")
|
||||
lines.append(f"links: {len(graph.get('links', []))}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def lint_wiki(wiki_dir: str) -> list[str]:
|
||||
"""检查 Wiki 中的孤立节点与稀疏页面。"""
|
||||
root = Path(wiki_dir)
|
||||
issues: list[str] = []
|
||||
|
||||
graph_path = root / "graph" / "edges.json"
|
||||
if graph_path.exists():
|
||||
graph = json.loads(graph_path.read_text(encoding="utf-8"))
|
||||
else:
|
||||
graph = {"nodes": [], "links": []}
|
||||
|
||||
linked_node_ids = {
|
||||
str(link.get("source", "")).strip()
|
||||
for link in graph.get("links", [])
|
||||
if str(link.get("source", "")).strip()
|
||||
}
|
||||
linked_node_ids.update(
|
||||
str(link.get("target", "")).strip()
|
||||
for link in graph.get("links", [])
|
||||
if str(link.get("target", "")).strip()
|
||||
)
|
||||
|
||||
for node in graph.get("nodes", []):
|
||||
node_id = str(node.get("id", "")).strip()
|
||||
if node_id and node_id not in linked_node_ids:
|
||||
label = str(node.get("label", "")).strip()
|
||||
issues.append(f"孤立节点: {node_id}" + (f" ({label})" if label else ""))
|
||||
|
||||
for _, path, _, body in _iter_entity_pages(wiki_dir):
|
||||
if len(body.strip()) < 50:
|
||||
relative_path = path.relative_to(root).as_posix()
|
||||
issues.append(f"稀疏页面: {relative_path} (正文 {len(body.strip())} 字符)")
|
||||
|
||||
return issues
|
||||
|
||||
|
||||
def ingest_paper(
|
||||
wiki_dir: str,
|
||||
arxiv_id: str | None = None,
|
||||
title: str | None = None,
|
||||
authors: str | None = None,
|
||||
year: str | None = None,
|
||||
venue: str = "arXiv",
|
||||
thesis: str = "",
|
||||
) -> Path | None:
|
||||
"""创建 paper 实体并补充标准化正文模板。"""
|
||||
abstract = ""
|
||||
authors_list: list[str]
|
||||
paper_title = title
|
||||
paper_year = year
|
||||
paper_arxiv_id = None
|
||||
|
||||
if arxiv_id:
|
||||
metadata = _fetch_arxiv_metadata(arxiv_id)
|
||||
if metadata is None:
|
||||
return None
|
||||
paper_title = metadata["title"]
|
||||
authors_list = metadata["authors"]
|
||||
paper_year = metadata["year"]
|
||||
abstract = metadata["abstract"]
|
||||
paper_arxiv_id = metadata["arxiv_id"]
|
||||
else:
|
||||
if not (title and authors and year):
|
||||
raise ValueError("非 arXiv 论文需提供 --title、--authors、--year")
|
||||
authors_list = [
|
||||
author.strip() for author in authors.split(",") if author.strip()
|
||||
]
|
||||
if not authors_list:
|
||||
raise ValueError("authors 不能为空")
|
||||
|
||||
if not paper_title:
|
||||
raise ValueError("论文标题不能为空")
|
||||
if not paper_year:
|
||||
raise ValueError("论文年份不能为空")
|
||||
if not authors_list:
|
||||
raise ValueError("论文作者不能为空")
|
||||
|
||||
slug = f"{_last_name(authors_list[0])}{paper_year}_{slugify(paper_title)[:30]}"
|
||||
extra_frontmatter: dict[str, Any] = {
|
||||
"authors": authors_list,
|
||||
"year": paper_year,
|
||||
"venue": venue,
|
||||
}
|
||||
if paper_arxiv_id is not None:
|
||||
extra_frontmatter["arxiv_id"] = paper_arxiv_id
|
||||
|
||||
path = add_entity(wiki_dir, "paper", slug, paper_title, extra_frontmatter)
|
||||
|
||||
content = path.read_text(encoding="utf-8")
|
||||
if "## 一句话论点" not in content:
|
||||
body = f"\n## 一句话论点\n\n{thesis or '待填写'}\n\n"
|
||||
if abstract:
|
||||
body += f"## 摘要\n\n> {' '.join(abstract.split())}\n\n"
|
||||
body += (
|
||||
"## 问题/空白\n\n"
|
||||
"## 方法\n\n"
|
||||
"## 关键结果\n\n"
|
||||
"## 局限性\n\n"
|
||||
"## 与本项目的相关性\n\n"
|
||||
)
|
||||
path.write_text(content + body, encoding="utf-8")
|
||||
|
||||
return path
|
||||
|
||||
|
||||
def init_wiki(wiki_dir: str) -> None:
|
||||
"""初始化研究 Wiki 的目录结构与基础文件。
|
||||
|
||||
参数:
|
||||
wiki_dir: Wiki 根目录。
|
||||
"""
|
||||
root = Path(wiki_dir)
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
for entity_dir in ENTITY_TYPES.values():
|
||||
(root / entity_dir).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
graph_dir = root / "graph"
|
||||
graph_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
_write_if_missing(
|
||||
root / "index.md", "# Research Wiki 索引\n\n> 自动生成，请勿手动编辑。\n"
|
||||
)
|
||||
_write_if_missing(root / "log.md", "# Wiki 变更日志\n\n")
|
||||
_write_if_missing(root / "gap_map.md", "# 领域空白汇总\n\n")
|
||||
_write_if_missing(
|
||||
root / "query_pack.md",
|
||||
"# Query Pack\n\n> 尚无数据。运行 research-lit 或 idea-creator 后自动生成。\n",
|
||||
)
|
||||
_write_if_missing(
|
||||
graph_dir / "edges.json",
|
||||
json.dumps({"directed": True, "nodes": [], "links": []}, ensure_ascii=False),
|
||||
)
|
||||
|
||||
append_log(wiki_dir, "Wiki 初始化完成")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""命令行入口。"""
|
||||
parser = argparse.ArgumentParser(description="研究 Wiki 工具")
|
||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
init_parser = subparsers.add_parser("init", help="初始化 Wiki")
|
||||
init_parser.add_argument("wiki_dir", help="Wiki 根目录")
|
||||
|
||||
log_parser = subparsers.add_parser("log", help="追加日志")
|
||||
log_parser.add_argument("wiki_dir", help="Wiki 根目录")
|
||||
log_parser.add_argument("message", nargs="+", help="日志内容")
|
||||
|
||||
entity_parser = subparsers.add_parser("add_entity", help="创建实体")
|
||||
entity_parser.add_argument("wiki_dir", help="Wiki 根目录")
|
||||
entity_parser.add_argument(
|
||||
"--type", required=True, choices=list(ENTITY_TYPES), help="实体类型"
|
||||
)
|
||||
entity_parser.add_argument("--id", required=True, help="实体 ID")
|
||||
entity_parser.add_argument("--title", required=True, help="实体标题")
|
||||
|
||||
edge_parser = subparsers.add_parser("add_edge", help="创建关系边")
|
||||
edge_parser.add_argument("wiki_dir", help="Wiki 根目录")
|
||||
edge_parser.add_argument(
|
||||
"--from", dest="from_id", required=True, help="起点节点 ID"
|
||||
)
|
||||
edge_parser.add_argument("--to", dest="to_id", required=True, help="终点节点 ID")
|
||||
edge_parser.add_argument(
|
||||
"--type", required=True, choices=EDGE_TYPES, dest="edge_type", help="边类型"
|
||||
)
|
||||
edge_parser.add_argument("--evidence", default="", help="证据说明")
|
||||
|
||||
ingest_parser = subparsers.add_parser("ingest_paper", help="导入论文")
|
||||
ingest_parser.add_argument("wiki_dir", help="Wiki 根目录")
|
||||
ingest_parser.add_argument("--arxiv-id", dest="arxiv_id", help="arXiv ID")
|
||||
ingest_parser.add_argument("--title", help="论文标题")
|
||||
ingest_parser.add_argument("--authors", help="作者，逗号分隔")
|
||||
ingest_parser.add_argument("--year", help="年份")
|
||||
ingest_parser.add_argument("--venue", default="arXiv", help="发表 venue")
|
||||
ingest_parser.add_argument("--thesis", default="", help="一句话论点")
|
||||
|
||||
rebuild_query_pack_parser = subparsers.add_parser(
|
||||
"rebuild_query_pack", help="重建 Query Pack"
|
||||
)
|
||||
rebuild_query_pack_parser.add_argument("wiki_dir", help="Wiki 根目录")
|
||||
|
||||
rebuild_index_parser = subparsers.add_parser("rebuild_index", help="重建索引")
|
||||
rebuild_index_parser.add_argument("wiki_dir", help="Wiki 根目录")
|
||||
|
||||
stats_parser = subparsers.add_parser("stats", help="输出统计信息")
|
||||
stats_parser.add_argument("wiki_dir", help="Wiki 根目录")
|
||||
|
||||
lint_parser = subparsers.add_parser("lint", help="检查 Wiki 问题")
|
||||
lint_parser.add_argument("wiki_dir", help="Wiki 根目录")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.command == "init":
|
||||
init_wiki(args.wiki_dir)
|
||||
return
|
||||
|
||||
if args.command == "log":
|
||||
append_log(args.wiki_dir, " ".join(args.message))
|
||||
return
|
||||
|
||||
if args.command == "add_entity":
|
||||
add_entity(args.wiki_dir, args.type, args.id, args.title)
|
||||
return
|
||||
|
||||
if args.command == "add_edge":
|
||||
add_edge(args.wiki_dir, args.from_id, args.to_id, args.edge_type, args.evidence)
|
||||
return
|
||||
|
||||
if args.command == "ingest_paper":
|
||||
ingest_paper(
|
||||
args.wiki_dir,
|
||||
arxiv_id=args.arxiv_id,
|
||||
title=args.title,
|
||||
authors=args.authors,
|
||||
year=args.year,
|
||||
venue=args.venue,
|
||||
thesis=args.thesis,
|
||||
)
|
||||
return
|
||||
|
||||
if args.command == "rebuild_query_pack":
|
||||
rebuild_query_pack(args.wiki_dir)
|
||||
return
|
||||
|
||||
if args.command == "rebuild_index":
|
||||
rebuild_index(args.wiki_dir)
|
||||
return
|
||||
|
||||
if args.command == "stats":
|
||||
print(get_stats(args.wiki_dir))
|
||||
return
|
||||
|
||||
if args.command == "lint":
|
||||
issues = lint_wiki(args.wiki_dir)
|
||||
print("\n".join(issues) if issues else "未发现问题")
|
||||
return
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,224 @@
|
||||
"""Semantic Scholar 搜索命令行工具。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
try:
|
||||
import requests
|
||||
except ImportError:
|
||||
print("警告: 未安装 requests，已跳过 Semantic Scholar CLI。", file=sys.stderr)
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
API_URL = "https://api.semanticscholar.org/graph/v1/paper/search"
|
||||
REQUEST_FIELDS = "title,authors,year,venue,citationCount,externalIds,tldr"
|
||||
REQUEST_INTERVAL_SECONDS = 1.0
|
||||
REQUEST_TIMEOUT_SECONDS = 30.0
|
||||
USER_AGENT = "semantic_scholar_fetch.py/1.0"
|
||||
_LAST_REQUEST_AT: float | None = None
|
||||
|
||||
|
||||
def _normalize_filter_values(values: list[str] | None) -> str | None:
|
||||
"""将过滤参数整理为 API 需要的逗号分隔字符串。"""
|
||||
if not values:
|
||||
return None
|
||||
|
||||
items: list[str] = []
|
||||
for value in values:
|
||||
for item in value.split(","):
|
||||
cleaned = item.strip()
|
||||
if cleaned:
|
||||
items.append(cleaned)
|
||||
|
||||
if not items:
|
||||
return None
|
||||
return ",".join(items)
|
||||
|
||||
|
||||
def _respect_rate_limit() -> None:
|
||||
"""确保连续请求之间至少间隔一秒。"""
|
||||
global _LAST_REQUEST_AT
|
||||
|
||||
now = time.monotonic()
|
||||
if _LAST_REQUEST_AT is not None:
|
||||
elapsed = now - _LAST_REQUEST_AT
|
||||
if elapsed < REQUEST_INTERVAL_SECONDS:
|
||||
time.sleep(REQUEST_INTERVAL_SECONDS - elapsed)
|
||||
|
||||
_LAST_REQUEST_AT = time.monotonic()
|
||||
|
||||
|
||||
def _clean_text(value: Any) -> str:
|
||||
"""压缩并清理任意文本值。"""
|
||||
return " ".join(str(value).split())
|
||||
|
||||
|
||||
def _normalize_authors(authors: Any) -> list[str]:
|
||||
"""将作者字段统一为作者姓名列表。"""
|
||||
normalized_authors: list[str] = []
|
||||
if not isinstance(authors, list):
|
||||
return normalized_authors
|
||||
|
||||
for author in authors:
|
||||
if isinstance(author, dict):
|
||||
name = author.get("name")
|
||||
if name:
|
||||
normalized_authors.append(_clean_text(name))
|
||||
elif author:
|
||||
normalized_authors.append(_clean_text(author))
|
||||
|
||||
return normalized_authors
|
||||
|
||||
|
||||
def _normalize_external_ids(external_ids: Any) -> dict[str, str]:
|
||||
"""将 externalIds 统一整理为字符串字典，并保留 arXiv 标识。"""
|
||||
normalized: dict[str, str] = {}
|
||||
if isinstance(external_ids, dict):
|
||||
for key, value in external_ids.items():
|
||||
if value is None:
|
||||
continue
|
||||
cleaned_value = _clean_text(value)
|
||||
if cleaned_value:
|
||||
normalized[str(key)] = cleaned_value
|
||||
|
||||
arxiv_id = (
|
||||
normalized.get("ArXiv") or normalized.get("arXiv") or normalized.get("arxiv")
|
||||
)
|
||||
if arxiv_id and "ArXiv" not in normalized:
|
||||
normalized["ArXiv"] = arxiv_id
|
||||
|
||||
return normalized
|
||||
|
||||
|
||||
def _normalize_tldr(tldr: Any) -> Any:
|
||||
"""保留 TLDR 字段原始结构，但去除明显的空字符串。"""
|
||||
if isinstance(tldr, dict):
|
||||
normalized_tldr: dict[str, Any] = {}
|
||||
for key, value in tldr.items():
|
||||
if isinstance(value, str):
|
||||
cleaned = _clean_text(value)
|
||||
if cleaned:
|
||||
normalized_tldr[key] = cleaned
|
||||
elif value is not None:
|
||||
normalized_tldr[key] = value
|
||||
return normalized_tldr or None
|
||||
return tldr
|
||||
|
||||
|
||||
def _normalize_paper(paper: dict[str, Any]) -> dict[str, Any]:
|
||||
"""将 API 返回的论文记录压缩成稳定的输出结构。"""
|
||||
return {
|
||||
"title": _clean_text(paper.get("title", "")),
|
||||
"authors": _normalize_authors(paper.get("authors")),
|
||||
"year": paper.get("year"),
|
||||
"venue": _clean_text(paper.get("venue", "")),
|
||||
"citationCount": paper.get("citationCount"),
|
||||
"externalIds": _normalize_external_ids(paper.get("externalIds")),
|
||||
"tldr": _normalize_tldr(paper.get("tldr")),
|
||||
}
|
||||
|
||||
|
||||
def _request_json(url: str, params: dict[str, Any]) -> dict[str, Any]:
|
||||
"""发起受限频率控制的 GET 请求并返回 JSON 对象。"""
|
||||
_respect_rate_limit()
|
||||
response = requests.get(
|
||||
url,
|
||||
params=params,
|
||||
headers={"User-Agent": USER_AGENT},
|
||||
timeout=REQUEST_TIMEOUT_SECONDS,
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
payload = response.json()
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError("Semantic Scholar 返回了非对象类型的 JSON")
|
||||
return payload
|
||||
|
||||
|
||||
def search_papers(
|
||||
query: str,
|
||||
max_results: int,
|
||||
fields_of_study: list[str] | None = None,
|
||||
publication_types: list[str] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""搜索 Semantic Scholar 论文并返回标准化结果。"""
|
||||
params: dict[str, Any] = {
|
||||
"query": query,
|
||||
"limit": max_results,
|
||||
"fields": REQUEST_FIELDS,
|
||||
}
|
||||
|
||||
normalized_fields_of_study = _normalize_filter_values(fields_of_study)
|
||||
if normalized_fields_of_study is not None:
|
||||
params["fieldsOfStudy"] = normalized_fields_of_study
|
||||
|
||||
normalized_publication_types = _normalize_filter_values(publication_types)
|
||||
if normalized_publication_types is not None:
|
||||
params["publicationTypes"] = normalized_publication_types
|
||||
|
||||
payload = _request_json(API_URL, params)
|
||||
papers = payload.get("data", [])
|
||||
if not isinstance(papers, list):
|
||||
raise ValueError("Semantic Scholar 响应缺少 data 列表")
|
||||
|
||||
normalized_papers: list[dict[str, Any]] = []
|
||||
for paper in papers:
|
||||
if isinstance(paper, dict):
|
||||
normalized_papers.append(_normalize_paper(paper))
|
||||
return normalized_papers
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
"""构建命令行参数解析器。"""
|
||||
parser = argparse.ArgumentParser(description="Semantic Scholar 搜索工具")
|
||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
search_parser = subparsers.add_parser("search", help="搜索论文")
|
||||
search_parser.add_argument("query", help="搜索关键词")
|
||||
search_parser.add_argument("--max", dest="max_results", type=int, default=10)
|
||||
search_parser.add_argument(
|
||||
"--fields-of-study",
|
||||
nargs="+",
|
||||
default=None,
|
||||
help="按学科领域过滤",
|
||||
)
|
||||
search_parser.add_argument(
|
||||
"--publication-types",
|
||||
nargs="+",
|
||||
default=None,
|
||||
help="按出版类型过滤",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
"""CLI 入口。"""
|
||||
parser = build_parser()
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
try:
|
||||
if args.command == "search":
|
||||
results = search_papers(
|
||||
query=args.query,
|
||||
max_results=args.max_results,
|
||||
fields_of_study=args.fields_of_study,
|
||||
publication_types=args.publication_types,
|
||||
)
|
||||
json.dump(results, sys.stdout, ensure_ascii=False, indent=2)
|
||||
sys.stdout.write("\n")
|
||||
return 0
|
||||
except (requests.RequestException, ValueError) as exc:
|
||||
print(f"错误: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
parser.error(f"未知命令: {args.command}")
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user