chore: track claude skills, tools, templates, reference code and research-wiki
- Add all claude skills (brainstorming, commit, debugging, TDD, etc.) - Add claude hooks (pre-commit-guard, post-edit-quality) - Add research templates (experiment plan, research brief, etc.) - Add claude tools (arxiv/semantic_scholar/openalex fetch, wiki, exa) - Add TRM4 reference implementation as algorithm fidelity baseline - Add research-wiki content (plans, index, graph, query_pack) - Update .gitignore to exclude .graphify_version runtime state
This commit is contained in:
@@ -0,0 +1,185 @@
|
||||
"""arXiv 搜索与 PDF 下载的独立命令行工具。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
import xml.etree.ElementTree as ET
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
|
||||
ATOM_NS = "http://www.w3.org/2005/Atom"
|
||||
ARXIV_API_URL = (
|
||||
"http://export.arxiv.org/api/query?search_query=all:{query}&max_results={max}"
|
||||
)
|
||||
ARXIV_PDF_URL = "https://arxiv.org/pdf/{arxiv_id}.pdf"
|
||||
USER_AGENT = "arxiv_fetch.py/1.0"
|
||||
MIN_PDF_SIZE = 10 * 1024
|
||||
DOWNLOAD_DELAY_SECONDS = 1.0
|
||||
|
||||
|
||||
def _clean_text(value: str) -> str:
|
||||
"""压缩空白并清理文本内容。"""
|
||||
return " ".join(value.split())
|
||||
|
||||
|
||||
def _normalize_arxiv_id(raw_id: str) -> str:
|
||||
"""将 arXiv 标识归一化为不带版本号的论文 ID。"""
|
||||
value = raw_id.strip()
|
||||
if value.startswith("http://") or value.startswith("https://"):
|
||||
value = value.rsplit("/", 1)[-1]
|
||||
if value.startswith("abs/") or value.startswith("pdf/"):
|
||||
value = value.split("/", 1)[1]
|
||||
if value.endswith(".pdf"):
|
||||
value = value[:-4]
|
||||
if value.startswith("arXiv:") or value.startswith("arxiv:"):
|
||||
value = value.split(":", 1)[1]
|
||||
if "v" in value:
|
||||
head, tail = value.rsplit("v", 1)
|
||||
if tail.isdigit():
|
||||
value = head
|
||||
return value
|
||||
|
||||
|
||||
def _request_text(url: str) -> bytes:
|
||||
"""发起 HTTP 请求并返回原始字节内容。"""
|
||||
request = Request(url, headers={"User-Agent": USER_AGENT})
|
||||
with urlopen(request) as response:
|
||||
return response.read()
|
||||
|
||||
|
||||
def _parse_arxiv_feed(xml_bytes: bytes) -> list[dict[str, Any]]:
|
||||
"""解析 arXiv Atom Feed 并提取论文条目。"""
|
||||
root = ET.fromstring(xml_bytes)
|
||||
namespace = {"atom": ATOM_NS}
|
||||
results: list[dict[str, Any]] = []
|
||||
|
||||
for entry in root.findall("atom:entry", namespace):
|
||||
title = _clean_text(
|
||||
entry.findtext("atom:title", default="", namespaces=namespace)
|
||||
)
|
||||
abstract = _clean_text(
|
||||
entry.findtext("atom:summary", default="", namespaces=namespace)
|
||||
)
|
||||
published = _clean_text(
|
||||
entry.findtext("atom:published", default="", namespaces=namespace)
|
||||
)
|
||||
if "T" in published:
|
||||
published = published.split("T", 1)[0]
|
||||
|
||||
authors: list[str] = []
|
||||
for author in entry.findall("atom:author", namespace):
|
||||
name = _clean_text(
|
||||
author.findtext("atom:name", default="", namespaces=namespace)
|
||||
)
|
||||
if name:
|
||||
authors.append(name)
|
||||
|
||||
categories: list[str] = []
|
||||
for category in entry.findall("atom:category", namespace):
|
||||
term = category.get("term", "").strip()
|
||||
if term and term not in categories:
|
||||
categories.append(term)
|
||||
|
||||
raw_id = entry.findtext("atom:id", default="", namespaces=namespace)
|
||||
arxiv_id = _normalize_arxiv_id(raw_id.rsplit("/", 1)[-1] if raw_id else "")
|
||||
|
||||
results.append(
|
||||
{
|
||||
"title": title,
|
||||
"authors": authors,
|
||||
"arxiv_id": arxiv_id,
|
||||
"abstract": abstract,
|
||||
"categories": categories,
|
||||
"published": published,
|
||||
}
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def search_arxiv(query: str, max_results: int) -> list[dict[str, Any]]:
|
||||
"""搜索 arXiv 并返回结构化论文结果。"""
|
||||
encoded_query = quote(query, safe="")
|
||||
url = ARXIV_API_URL.format(query=encoded_query, max=max_results)
|
||||
feed_bytes = _request_text(url)
|
||||
return _parse_arxiv_feed(feed_bytes)
|
||||
|
||||
|
||||
def download_arxiv_pdf(arxiv_id: str, target_dir: Path) -> Path:
|
||||
"""下载指定 arXiv 论文的 PDF 并校验文件大小。"""
|
||||
normalized_id = _normalize_arxiv_id(arxiv_id)
|
||||
target_dir.mkdir(parents=True, exist_ok=True)
|
||||
output_path = target_dir / f"{normalized_id}.pdf"
|
||||
pdf_url = ARXIV_PDF_URL.format(arxiv_id=normalized_id)
|
||||
|
||||
request = Request(pdf_url, headers={"User-Agent": USER_AGENT})
|
||||
time.sleep(DOWNLOAD_DELAY_SECONDS)
|
||||
try:
|
||||
with urlopen(request) as response, output_path.open("wb") as handle:
|
||||
while True:
|
||||
chunk = response.read(8192)
|
||||
if not chunk:
|
||||
break
|
||||
handle.write(chunk)
|
||||
except Exception:
|
||||
if output_path.exists():
|
||||
output_path.unlink()
|
||||
raise
|
||||
|
||||
if output_path.stat().st_size <= MIN_PDF_SIZE:
|
||||
output_path.unlink(missing_ok=True)
|
||||
raise ValueError(f"下载文件过小，疑似失败: {output_path}")
|
||||
|
||||
return output_path
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
"""构建命令行参数解析器。"""
|
||||
parser = argparse.ArgumentParser(description="arXiv 搜索与 PDF 下载工具")
|
||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
search_parser = subparsers.add_parser("search", help="搜索 arXiv 论文")
|
||||
search_parser.add_argument("query", help="搜索关键词")
|
||||
search_parser.add_argument("--max", type=int, default=10, help="最大返回数量")
|
||||
|
||||
download_parser = subparsers.add_parser("download", help="下载 arXiv PDF")
|
||||
download_parser.add_argument("arxiv_id", help="arXiv 论文 ID")
|
||||
download_parser.add_argument(
|
||||
"--dir", dest="target_dir", default=".", help="保存目录"
|
||||
)
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
"""CLI 入口。"""
|
||||
parser = build_parser()
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
try:
|
||||
if args.command == "search":
|
||||
results = search_arxiv(args.query, args.max)
|
||||
json.dump(results, sys.stdout, ensure_ascii=False, indent=2)
|
||||
sys.stdout.write("\n")
|
||||
return 0
|
||||
|
||||
if args.command == "download":
|
||||
output_path = download_arxiv_pdf(args.arxiv_id, Path(args.target_dir))
|
||||
sys.stdout.write(f"{output_path}\n")
|
||||
return 0
|
||||
except Exception as exc:
|
||||
print(f"错误: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
parser.error(f"未知命令: {args.command}")
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user