chore: track claude skills, tools, templates, reference code and research-wiki
- Add all claude skills (brainstorming, commit, debugging, TDD, etc.) - Add claude hooks (pre-commit-guard, post-edit-quality) - Add research templates (experiment plan, research brief, etc.) - Add claude tools (arxiv/semantic_scholar/openalex fetch, wiki, exa) - Add TRM4 reference implementation as algorithm fidelity baseline - Add research-wiki content (plans, index, graph, query_pack) - Update .gitignore to exclude .graphify_version runtime state
This commit is contained in:
@@ -0,0 +1,138 @@
|
||||
"""
|
||||
_download_meta.py — VideoMME 元数据下载与长视频列表提取
|
||||
==========================================================
|
||||
从 HuggingFace `lmms-lab/Video-MME` 下载数据集元数据，
|
||||
过滤 duration_category == "long"（30-60 分钟）的视频，
|
||||
输出两个文件：
|
||||
- {meta_dir}/long_videos.jsonl 每行一条唯一长视频记录
|
||||
- {meta_dir}/long_videos_qa.jsonl 每行一条 QA 对（含 video_id）
|
||||
|
||||
使用方式（由 build_videomme_trees.sh 调用）:
|
||||
python _download_meta.py --meta-dir /data/videomme/metadata
|
||||
|
||||
依赖（在脚本中已安装）: datasets, huggingface_hub
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
p = argparse.ArgumentParser(description="下载 VideoMME 元数据并提取长视频列表")
|
||||
p.add_argument("--meta-dir", required=True, help="元数据输出目录")
|
||||
p.add_argument(
|
||||
"--min-duration",
|
||||
type=int,
|
||||
default=1800,
|
||||
help="最短视频时长（秒），默认 1800（30 分钟）",
|
||||
)
|
||||
p.add_argument(
|
||||
"--max-duration",
|
||||
type=int,
|
||||
default=3600,
|
||||
help="最长视频时长（秒），默认 3600（60 分钟）",
|
||||
)
|
||||
return p.parse_args()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
meta_dir = Path(args.meta_dir)
|
||||
meta_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Phase 1: 尝试加载 HuggingFace 数据集
|
||||
print("[meta] 正在从 HuggingFace 加载 lmms-lab/Video-MME ...")
|
||||
try:
|
||||
from datasets import load_dataset # type: ignore
|
||||
except ImportError:
|
||||
print("[meta][ERROR] 未安装 datasets 库，请先运行: pip install datasets", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
# VideoMME 数据集，test split
|
||||
ds = load_dataset("lmms-lab/Video-MME", split="test")
|
||||
except Exception as e:
|
||||
print(f"[meta][ERROR] 数据集加载失败: {e}", file=sys.stderr)
|
||||
print("[meta] 请确认 HuggingFace 可访问，或配置 HF_ENDPOINT 镜像", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
print(f"[meta] 数据集总条目数: {len(ds)}")
|
||||
|
||||
# Phase 2: 过滤长视频
|
||||
# VideoMME 字段: video_id, youtube_id, url, duration, duration_category,
|
||||
# domain, sub_category, question, answer, options
|
||||
seen_video_ids: set[str] = set()
|
||||
long_videos: list[dict] = []
|
||||
long_qa: list[dict] = []
|
||||
|
||||
for row in ds:
|
||||
# 实际字段结构（lmms-lab/Video-MME 真实格式）:
|
||||
# video_id : "001", "002", ... （数据集内部序号）
|
||||
# videoID : YouTube 视频 ID（如 "fFjv93ACGo8"）
|
||||
# url : YouTube 完整链接
|
||||
# duration : "short" | "medium" | "long"（字符串类别，非秒数）
|
||||
# domain : 领域
|
||||
# sub_category : 细分类别
|
||||
# question_id : 问题序号
|
||||
# question : 问题文本
|
||||
# options : 选项列表（字符串）
|
||||
# answer : 正确选项字母
|
||||
duration_category = str(row.get("duration", "")).strip().lower()
|
||||
if duration_category != "long":
|
||||
continue
|
||||
|
||||
youtube_id = row.get("videoID") or row.get("video_id", "")
|
||||
url = row.get("url", f"https://www.youtube.com/watch?v={youtube_id}")
|
||||
|
||||
# 唯一视频记录（以 youtube_id 去重）
|
||||
if youtube_id not in seen_video_ids:
|
||||
seen_video_ids.add(youtube_id)
|
||||
long_videos.append(
|
||||
{
|
||||
"video_id": row.get("video_id", ""),
|
||||
"youtube_id": youtube_id,
|
||||
"url": url,
|
||||
"duration_category": duration_category,
|
||||
"domain": row.get("domain", ""),
|
||||
"sub_category": row.get("sub_category", ""),
|
||||
}
|
||||
)
|
||||
|
||||
# QA 对记录
|
||||
long_qa.append(
|
||||
{
|
||||
"video_id": row.get("video_id", ""),
|
||||
"youtube_id": youtube_id,
|
||||
"question_id": row.get("question_id", ""),
|
||||
"question": row.get("question", ""),
|
||||
"answer": row.get("answer", ""),
|
||||
"options": row.get("options", []),
|
||||
"duration_category": duration_category,
|
||||
}
|
||||
)
|
||||
|
||||
# Phase 3: 写出文件
|
||||
videos_path = meta_dir / "long_videos.jsonl"
|
||||
qa_path = meta_dir / "long_videos_qa.jsonl"
|
||||
|
||||
with open(videos_path, "w", encoding="utf-8") as f:
|
||||
for v in long_videos:
|
||||
f.write(json.dumps(v, ensure_ascii=False) + "\n")
|
||||
|
||||
with open(qa_path, "w", encoding="utf-8") as f:
|
||||
for q in long_qa:
|
||||
f.write(json.dumps(q, ensure_ascii=False) + "\n")
|
||||
|
||||
print(f"[meta] 长视频唯一数量: {len(long_videos)}")
|
||||
print(f"[meta] 长视频 QA 对数: {len(long_qa)}")
|
||||
print(f"[meta] 视频列表已保存: {videos_path}")
|
||||
print(f"[meta] QA 列表已保存: {qa_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,60 @@
|
||||
"""
|
||||
单视频建树脚本（仅 VLM，不加载 EmbeddingModel）
|
||||
================================================
|
||||
直接调用 VideoTreeBuilder，跳过 Pipeline 的嵌入模型初始化。
|
||||
结果保存为 JSON 到 cache/trees/ 目录。
|
||||
|
||||
用法::
|
||||
|
||||
conda run -n Video-Tree-TRM python scripts/build_tree_single.py \
|
||||
--video data/videomme/videos/xKiRmesHWIA.mp4
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# 项目根目录加入 sys.path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from video_tree_trm.config import Config
|
||||
from video_tree_trm.llm_client import LLMClient
|
||||
from video_tree_trm.video_tree_builder import VideoTreeBuilder
|
||||
from utils.logger_system import log_msg
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""构建单个视频的 TreeIndex，仅使用 VLM，不加载 EmbeddingModel。"""
|
||||
parser = argparse.ArgumentParser(description="单视频建树（仅 VLM）")
|
||||
parser.add_argument("--video", required=True, help="视频文件路径")
|
||||
parser.add_argument("--config", default="config/default.yaml", help="配置文件路径")
|
||||
args = parser.parse_args()
|
||||
|
||||
# Phase 1: 加载配置 + 初始化 VLM
|
||||
cfg = Config.load(args.config)
|
||||
vlm = LLMClient(cfg.vlm)
|
||||
|
||||
# Phase 2: 构建树（纯 VLM 描述，embedding=None）
|
||||
builder = VideoTreeBuilder(vlm, cfg.tree)
|
||||
tree = builder.build(args.video)
|
||||
|
||||
# Phase 3: 保存 JSON
|
||||
stem = Path(args.video).stem
|
||||
cache_dir = Path(cfg.tree.cache_dir)
|
||||
cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
out_path = str(cache_dir / f"{stem}_video.json")
|
||||
tree.save_json(out_path)
|
||||
|
||||
log_msg("INFO", "建树完成，已保存", path=out_path)
|
||||
print(f"\n[完成] TreeIndex 已保存到: {out_path}")
|
||||
print(f" L1 节点数: {len(tree.roots)}")
|
||||
total_l2 = sum(len(r.children) for r in tree.roots)
|
||||
total_l3 = sum(len(l2.children) for r in tree.roots for l2 in r.children)
|
||||
print(f" L2 节点数: {total_l2}")
|
||||
print(f" L3 节点数: {total_l3}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,224 @@
|
||||
"""
|
||||
批量视频建树脚本（视频间并行 + 视频内 asyncio 真并发）
|
||||
==========================================================
|
||||
扫描指定目录下所有 MP4，跳过已有 JSON 树的视频，
|
||||
使用 ThreadPoolExecutor 进行视频间并行（--jobs，默认 1），
|
||||
每个视频内部使用 asyncio + Semaphore(concurrency=16) 真并发 VLM 调用。
|
||||
|
||||
并发架构::
|
||||
|
||||
外层: ThreadPoolExecutor(max_workers=jobs=1) — 视频间并行（默认 1 路）
|
||||
内层: asyncio.run(_build_async()) — 每视频独立事件循环
|
||||
asyncio.Semaphore(concurrency=16) — 限制同时在途 VLM 请求数
|
||||
总最大 VLM 并发: jobs × concurrency = 1 × 16 = 16
|
||||
|
||||
用法::
|
||||
|
||||
# 默认 1 路 16 并发
|
||||
conda run -n Video-Tree-TRM python scripts/build_trees_batch.py
|
||||
|
||||
# 指定路数
|
||||
conda run -n Video-Tree-TRM python scripts/build_trees_batch.py --jobs 2
|
||||
|
||||
# 指定目录和配置
|
||||
conda run -n Video-Tree-TRM python scripts/build_trees_batch.py \\
|
||||
--video-dir data/videomme/videos \\
|
||||
--config config/videomme.yaml \\
|
||||
--jobs 3
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import time
|
||||
from concurrent.futures import FIRST_COMPLETED, Future, ThreadPoolExecutor
|
||||
from concurrent.futures import wait as cfwait
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
# 项目根目录加入 sys.path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from video_tree_trm.config import Config
|
||||
from video_tree_trm.llm_client import LLMClient
|
||||
from video_tree_trm.video_tree_builder import VideoTreeBuilder
|
||||
from utils.logger_system import log_msg
|
||||
|
||||
|
||||
def _build_one(
|
||||
video_path: Path,
|
||||
cfg: Config,
|
||||
) -> Tuple[str, bool, str]:
|
||||
"""构建单个视频的 TreeIndex 并保存 JSON。
|
||||
|
||||
参数:
|
||||
video_path: 视频文件绝对路径。
|
||||
cfg: 已加载的配置对象（由主进程共享，仅读取）。
|
||||
|
||||
返回:
|
||||
(stem, success, message) 三元组。
|
||||
|
||||
实现细节:
|
||||
每次调用独立初始化 LLMClient（避免多线程共享同一 httpx.Client 内部状态），
|
||||
使用 VideoTreeBuilder.build() 内部的异步事件循环（L2→L3→L1 链式并发）。
|
||||
"""
|
||||
stem = video_path.stem
|
||||
try:
|
||||
progress_dir = Path(cfg.tree.cache_dir) / "progress"
|
||||
progress_path = progress_dir / f"{stem}.json"
|
||||
if progress_path.is_file():
|
||||
log_msg("INFO", "检测到中间进度，启用断点续跑", stem=stem, progress_path=str(progress_path))
|
||||
|
||||
# 每线程独立 LLMClient（httpx.Client 线程安全，但独立更稳健）
|
||||
vlm = LLMClient(cfg.vlm)
|
||||
builder = VideoTreeBuilder(vlm, cfg.tree)
|
||||
tree = builder.build(str(video_path))
|
||||
|
||||
# 保存 JSON
|
||||
cache_dir = Path(cfg.tree.cache_dir)
|
||||
cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
out_path = cache_dir / f"{stem}_video.json"
|
||||
tree.save_json(str(out_path))
|
||||
|
||||
l1 = len(tree.roots)
|
||||
l2 = sum(len(r.children) for r in tree.roots)
|
||||
l3 = sum(len(l2n.children) for r in tree.roots for l2n in r.children)
|
||||
msg = f"L1={l1} L2={l2} L3={l3} → {out_path}"
|
||||
log_msg("INFO", "视频建树完成", stem=stem, l1=l1, l2=l2, l3=l3)
|
||||
return stem, True, msg
|
||||
|
||||
except Exception as e: # noqa: BLE001
|
||||
log_msg("ERROR", "视频建树失败", stem=stem, error=str(e))
|
||||
return stem, False, str(e)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""批量建树主函数：视频间 ThreadPoolExecutor 并行 + 视频内异步事件循环。"""
|
||||
parser = argparse.ArgumentParser(description="批量视频建树（视频间并行）")
|
||||
parser.add_argument(
|
||||
"--video-dir",
|
||||
default="data/videomme/videos",
|
||||
help="MP4 视频目录（默认: data/videomme/videos）",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--config",
|
||||
default="config/videomme.yaml",
|
||||
help="配置文件路径（默认: config/videomme.yaml）",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--jobs",
|
||||
type=int,
|
||||
default=1,
|
||||
help="视频间并行数（默认: 1，每路视频内 asyncio Semaphore(concurrency) 并发 VLM）",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
# Phase 1: 加载配置（所有线程共享，只读）
|
||||
cfg = Config.load(args.config)
|
||||
log_msg(
|
||||
"INFO",
|
||||
"批量建树配置",
|
||||
video_dir=args.video_dir,
|
||||
cache_dir=cfg.tree.cache_dir,
|
||||
jobs=args.jobs,
|
||||
intra_concurrency=cfg.tree.concurrency,
|
||||
)
|
||||
|
||||
# Phase 2: 扫描视频 + 过滤已有树
|
||||
video_dir = Path(args.video_dir)
|
||||
assert video_dir.is_dir(), f"视频目录不存在: {video_dir}"
|
||||
all_videos: List[Path] = sorted(video_dir.glob("*.mp4"))
|
||||
|
||||
cache_dir = Path(cfg.tree.cache_dir)
|
||||
progress_dir = cache_dir / "progress"
|
||||
with_progress: List[Path] = []
|
||||
without_progress: List[Path] = []
|
||||
for v in all_videos:
|
||||
if (cache_dir / f"{v.stem}_video.json").exists():
|
||||
continue
|
||||
progress_path = progress_dir / f"{v.stem}.json"
|
||||
if progress_path.is_file():
|
||||
with_progress.append(v)
|
||||
else:
|
||||
without_progress.append(v)
|
||||
pending: List[Path] = with_progress + without_progress
|
||||
skipped = len(all_videos) - len(pending)
|
||||
|
||||
print(f"\n===== 批量建树 =====")
|
||||
print(f" 总视频数: {len(all_videos)}")
|
||||
print(f" 已跳过(已建): {skipped}")
|
||||
print(f" 待处理: {len(pending)}")
|
||||
print(f" 视频间并行: {args.jobs}")
|
||||
print(f" 视频内并发: {cfg.tree.concurrency}")
|
||||
print(f" 输出目录: {cache_dir}\n")
|
||||
|
||||
if not pending:
|
||||
print("所有视频均已建树，无需处理。")
|
||||
return
|
||||
|
||||
# Phase 3: 异步视频间并行（ThreadPoolExecutor + FIRST_COMPLETED 事件循环）
|
||||
built: List[str] = []
|
||||
failed: List[Tuple[str, str]] = []
|
||||
start_total = time.time()
|
||||
|
||||
pending_futures: Dict[Future, Path] = {}
|
||||
pending_queue = list(pending) # 待提交队列
|
||||
pool = ThreadPoolExecutor(max_workers=args.jobs)
|
||||
|
||||
# 初始填满 jobs 个任务
|
||||
while pending_queue and len(pending_futures) < args.jobs:
|
||||
video = pending_queue.pop(0)
|
||||
fut = pool.submit(_build_one, video, cfg)
|
||||
pending_futures[fut] = video
|
||||
print(f"[提交] {video.stem} ({len(built)+len(failed)+len(pending_futures)}/{len(pending)})")
|
||||
|
||||
# 事件循环：完成一个，补一个
|
||||
done_count = 0
|
||||
while pending_futures:
|
||||
done, _ = cfwait(list(pending_futures), return_when=FIRST_COMPLETED)
|
||||
for fut in done:
|
||||
video = pending_futures.pop(fut)
|
||||
stem, success, msg = fut.result()
|
||||
done_count += 1
|
||||
elapsed = time.time() - start_total
|
||||
if success:
|
||||
built.append(stem)
|
||||
print(f"[OK {done_count}/{len(pending)}] {stem} {msg} (累计 {elapsed:.0f}s)")
|
||||
else:
|
||||
failed.append((stem, msg))
|
||||
print(f"[FAIL {done_count}/{len(pending)}] {stem} {msg}")
|
||||
|
||||
# 补充下一个任务（非阻塞提交）
|
||||
if pending_queue:
|
||||
next_video = pending_queue.pop(0)
|
||||
next_fut = pool.submit(_build_one, next_video, cfg)
|
||||
pending_futures[next_fut] = next_video
|
||||
print(f"[提交] {next_video.stem} ({done_count+len(pending_futures)}/{len(pending)})")
|
||||
|
||||
pool.shutdown(wait=False)
|
||||
|
||||
# Phase 4: 汇总
|
||||
total_elapsed = time.time() - start_total
|
||||
print(f"\n===== 汇总 =====")
|
||||
print(f" 成功: {len(built)}")
|
||||
print(f" 失败: {len(failed)}")
|
||||
print(f" 跳过: {skipped}")
|
||||
print(f" 总耗时: {total_elapsed:.1f}s")
|
||||
if failed:
|
||||
print("\n 失败列表:")
|
||||
for stem, err in failed:
|
||||
print(f" {stem}: {err}")
|
||||
|
||||
log_msg(
|
||||
"INFO",
|
||||
"批量建树完成",
|
||||
built=len(built),
|
||||
failed=len(failed),
|
||||
skipped=skipped,
|
||||
elapsed_s=round(total_elapsed, 1),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+87
@@ -0,0 +1,87 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# build_trees_from_mp4.sh — 从本地 MP4 批量建树（JSON 输出，无 embedding）
|
||||
#
|
||||
# 用法:
|
||||
# bash scripts/build_trees_from_mp4.sh
|
||||
# bash scripts/build_trees_from_mp4.sh data/videomme/videos/TGom0uiW130.mp4 # 单文件
|
||||
#
|
||||
# 环境变量:
|
||||
# VIDEO_DIR — MP4 目录（默认: <PROJECT_ROOT>/data/videomme/videos）
|
||||
# CONFIG — 配置文件（默认: config/videomme.yaml）
|
||||
# =============================================================================
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
|
||||
VIDEO_DIR="${VIDEO_DIR:-${PROJECT_ROOT}/data/videomme/videos}"
|
||||
CONFIG="${CONFIG:-${PROJECT_ROOT}/config/videomme.yaml}"
|
||||
TREE_DIR="${PROJECT_ROOT}/data/videomme/trees"
|
||||
LOG_DIR="${PROJECT_ROOT}/data/videomme/logs"
|
||||
LOG="${LOG_DIR}/mp4_build_$(date +%Y%m%d_%H%M%S).log"
|
||||
FAILED="${LOG_DIR}/failed_mp4_builds.txt"
|
||||
|
||||
mkdir -p "${TREE_DIR}" "${LOG_DIR}"
|
||||
|
||||
# 绕过代理
|
||||
export NO_PROXY="${NO_PROXY:+${NO_PROXY},}100.83.164.94"
|
||||
export no_proxy="${no_proxy:+${no_proxy},}100.83.164.94"
|
||||
|
||||
# 激活 conda 环境
|
||||
CONDA_BASE="$(conda info --base 2>/dev/null || echo "${HOME}/miniconda3")"
|
||||
# shellcheck source=/dev/null
|
||||
source "${CONDA_BASE}/etc/profile.d/conda.sh"
|
||||
conda activate Video-Tree-TRM
|
||||
cd "${PROJECT_ROOT}"
|
||||
|
||||
# 确定待处理文件列表
|
||||
if [[ $# -gt 0 ]]; then
|
||||
MP4_FILES=("$@")
|
||||
else
|
||||
mapfile -t MP4_FILES < <(find "${VIDEO_DIR}" -name "*.mp4" | sort)
|
||||
fi
|
||||
|
||||
TOTAL=${#MP4_FILES[@]}
|
||||
BUILT=0; SKIP=0; FAIL=0
|
||||
|
||||
echo "[$(date)] ===== MP4 本地建树开始 =====" | tee "${LOG}"
|
||||
echo "[$(date)] 待处理: ${TOTAL} 个视频" | tee -a "${LOG}"
|
||||
echo "[$(date)] 配置: ${CONFIG}" | tee -a "${LOG}"
|
||||
|
||||
for MP4 in "${MP4_FILES[@]}"; do
|
||||
[[ -z "${MP4}" ]] && continue
|
||||
STEM="$(basename "${MP4}" .mp4)"
|
||||
CACHE="${TREE_DIR}/${STEM}_video.json"
|
||||
|
||||
# 缓存命中跳过
|
||||
if [[ -f "${CACHE}" ]]; then
|
||||
SKIP=$((SKIP+1))
|
||||
echo "[$(date)] [SKIP] ${STEM}" | tee -a "${LOG}"
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "[$(date)] [BUILD] ${STEM} file=${MP4}" | tee -a "${LOG}"
|
||||
|
||||
if python -u main.py index \
|
||||
--source "${MP4}" \
|
||||
--modality video \
|
||||
--config "${CONFIG}" \
|
||||
>> "${LOG}" 2>&1; then
|
||||
BUILT=$((BUILT+1))
|
||||
echo "[$(date)] [OK] ${STEM}" | tee -a "${LOG}"
|
||||
else
|
||||
FAIL=$((FAIL+1))
|
||||
echo "[$(date)] [FAIL] ${STEM}" | tee -a "${LOG}"
|
||||
echo "${STEM} ${MP4}" >> "${FAILED}"
|
||||
fi
|
||||
done
|
||||
|
||||
TREE_COUNT=$(find "${TREE_DIR}" -name "*_video.json" 2>/dev/null | wc -l)
|
||||
echo "" | tee -a "${LOG}"
|
||||
echo "[$(date)] ===== 汇总 =====" | tee -a "${LOG}"
|
||||
echo "[$(date)] 本次新建: ${BUILT} 跳过: ${SKIP} 失败: ${FAIL}" | tee -a "${LOG}"
|
||||
echo "[$(date)] 树索引总数: ${TREE_COUNT}" | tee -a "${LOG}"
|
||||
[[ ${FAIL} -gt 0 ]] && echo "[$(date)] 失败列表: ${FAILED}" | tee -a "${LOG}"
|
||||
echo "[$(date)] 日志: ${LOG}" | tee -a "${LOG}"
|
||||
Executable
+125
@@ -0,0 +1,125 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# build_trees_from_urls.sh — 直接从 YouTube URL 批量建树（不下载视频）
|
||||
#
|
||||
# 用法:
|
||||
# # 全量（300 个长视频）
|
||||
# bash scripts/build_trees_from_urls.sh
|
||||
#
|
||||
# # 只处理前 N 个
|
||||
# head -10 data/videomme/metadata/long_videos.jsonl \
|
||||
# | bash scripts/build_trees_from_urls.sh --stdin
|
||||
#
|
||||
# 环境变量:
|
||||
# DATA_DIR — 数据根目录（默认: <PROJECT_ROOT>/data/videomme）
|
||||
# CONFIG — 配置文件路径（默认: config/videomme.yaml）
|
||||
#
|
||||
# 特性:
|
||||
# - 自动激活 Video-Tree-TRM conda 环境
|
||||
# - 缓存命中跳过（trees/{youtube_id}_video.pkl 已存在则跳过）
|
||||
# - 断点续传（重复运行安全）
|
||||
# - 失败记录写入 logs/failed_url_builds.txt
|
||||
# =============================================================================
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 0. 全局配置
|
||||
# ---------------------------------------------------------------------------
|
||||
DATA_DIR="${DATA_DIR:-${PROJECT_ROOT}/data/videomme}"
|
||||
CONFIG="${CONFIG:-${PROJECT_ROOT}/config/videomme.yaml}"
|
||||
JSONL="${DATA_DIR}/metadata/long_videos.jsonl"
|
||||
TREE_DIR="${DATA_DIR}/trees"
|
||||
LOG_DIR="${DATA_DIR}/logs"
|
||||
LOG="${LOG_DIR}/url_build_$(date +%Y%m%d_%H%M%S).log"
|
||||
FAILED="${LOG_DIR}/failed_url_builds.txt"
|
||||
|
||||
mkdir -p "${TREE_DIR}" "${LOG_DIR}"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 0.5 绕过代理：GPU 内网地址直连，不经过 http_proxy
|
||||
# ---------------------------------------------------------------------------
|
||||
export NO_PROXY="${NO_PROXY:+${NO_PROXY},}100.83.164.94"
|
||||
export no_proxy="${no_proxy:+${no_proxy},}100.83.164.94"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 1. 激活 conda 环境
|
||||
# ---------------------------------------------------------------------------
|
||||
CONDA_BASE="$(conda info --base 2>/dev/null || echo "${HOME}/miniconda3")"
|
||||
# shellcheck source=/dev/null
|
||||
source "${CONDA_BASE}/etc/profile.d/conda.sh"
|
||||
conda activate Video-Tree-TRM
|
||||
cd "${PROJECT_ROOT}"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 2. 解析参数
|
||||
# ---------------------------------------------------------------------------
|
||||
STDIN_MODE=false
|
||||
for arg in "$@"; do
|
||||
[[ "${arg}" == "--stdin" ]] && STDIN_MODE=true
|
||||
done
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 3. 读取输入
|
||||
# ---------------------------------------------------------------------------
|
||||
if [[ "${STDIN_MODE}" == true ]]; then
|
||||
# 从 stdin 读取（支持 head -N | bash ... --stdin）
|
||||
INPUT_DATA="$(cat)"
|
||||
else
|
||||
INPUT_DATA="$(cat "${JSONL}")"
|
||||
fi
|
||||
|
||||
TOTAL=$(echo "${INPUT_DATA}" | wc -l)
|
||||
BUILT=0; SKIP=0; FAIL=0
|
||||
|
||||
echo "[$(date)] ===== URL 流式建树开始 =====" | tee "${LOG}"
|
||||
echo "[$(date)] 待处理: ${TOTAL} 个长视频" | tee -a "${LOG}"
|
||||
echo "[$(date)] 配置: ${CONFIG}" | tee -a "${LOG}"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 4. 主循环
|
||||
# ---------------------------------------------------------------------------
|
||||
while IFS= read -r line; do
|
||||
[[ -z "${line}" ]] && continue
|
||||
|
||||
YID=$(python -c "import sys,json; print(json.loads(sys.argv[1])['youtube_id'])" "${line}")
|
||||
URL=$(python -c "import sys,json; print(json.loads(sys.argv[1])['url'])" "${line}")
|
||||
CACHE="${TREE_DIR}/${YID}_video.json"
|
||||
|
||||
# 缓存命中跳过
|
||||
if [[ -f "${CACHE}" ]]; then
|
||||
SKIP=$((SKIP+1))
|
||||
echo "[$(date)] [SKIP] ${YID}" | tee -a "${LOG}"
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "[$(date)] [BUILD] ${YID} url=${URL}" | tee -a "${LOG}"
|
||||
|
||||
if python main.py index \
|
||||
--source "${URL}" \
|
||||
--modality video \
|
||||
--config "${CONFIG}" \
|
||||
>> "${LOG}" 2>&1; then
|
||||
BUILT=$((BUILT+1))
|
||||
echo "[$(date)] [OK] ${YID}" | tee -a "${LOG}"
|
||||
else
|
||||
FAIL=$((FAIL+1))
|
||||
echo "[$(date)] [FAIL] ${YID}" | tee -a "${LOG}"
|
||||
echo "${YID} ${URL}" >> "${FAILED}"
|
||||
fi
|
||||
|
||||
done <<< "${INPUT_DATA}"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 5. 汇总
|
||||
# ---------------------------------------------------------------------------
|
||||
TREE_COUNT=$(find "${TREE_DIR}" -name "*_video.json" 2>/dev/null | wc -l)
|
||||
echo "" | tee -a "${LOG}"
|
||||
echo "[$(date)] ===== 汇总 =====" | tee -a "${LOG}"
|
||||
echo "[$(date)] 本次新建: ${BUILT} 跳过: ${SKIP} 失败: ${FAIL}" | tee -a "${LOG}"
|
||||
echo "[$(date)] 树索引总数: ${TREE_COUNT} / ${TOTAL}" | tee -a "${LOG}"
|
||||
[[ ${FAIL} -gt 0 ]] && echo "[$(date)] 失败列表: ${FAILED}" | tee -a "${LOG}"
|
||||
echo "[$(date)] 日志: ${LOG}" | tee -a "${LOG}"
|
||||
Executable
+320
@@ -0,0 +1,320 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# build_videomme_trees.sh — VideoMME 长视频数据预处理：下载 + 建树
|
||||
# =============================================================================
|
||||
# 功能:
|
||||
# 1. 初始化目录结构 (/data/videomme/...)
|
||||
# 2. 激活 Conda 环境 (Video-Tree-TRM)
|
||||
# 3. 安装必要工具 (yt-dlp, datasets)
|
||||
# 4. 从 HuggingFace 下载 VideoMME 元数据，提取长视频列表
|
||||
# 5. 用 yt-dlp 批量下载长视频（断点续传，跳过已下载）
|
||||
# 6. 为每个视频调用 main.py index 建树（跳过已缓存）
|
||||
# 7. 汇总日志
|
||||
#
|
||||
# 使用方式:
|
||||
# cd /home/undergraduate/Video-Tree-TRM
|
||||
# bash scripts/build_videomme_trees.sh
|
||||
#
|
||||
# 可选环境变量覆盖:
|
||||
# DATA_DIR=/other/path bash scripts/build_videomme_trees.sh
|
||||
# WORKERS=4 bash scripts/build_videomme_trees.sh # 并行建树进程数
|
||||
#
|
||||
# 断点续传:
|
||||
# 重复运行完全安全 —— yt-dlp 跳过已下载文件，main.py 跳过缓存命中的树。
|
||||
# =============================================================================
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 0. 全局配置
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
|
||||
CONDA_ENV="${CONDA_ENV:-Video-Tree-TRM}"
|
||||
DATA_DIR="${DATA_DIR:-${PROJECT_ROOT}/data/videomme}"
|
||||
VIDEO_DIR="${DATA_DIR}/videos"
|
||||
META_DIR="${DATA_DIR}/metadata"
|
||||
TREE_DIR="${DATA_DIR}/trees"
|
||||
LOG_DIR="${DATA_DIR}/logs"
|
||||
CKPT_DIR="${DATA_DIR}/checkpoints"
|
||||
|
||||
CONFIG_YAML="${PROJECT_ROOT}/config/videomme.yaml"
|
||||
ENV_FILE="${PROJECT_ROOT}/.env"
|
||||
META_SCRIPT="${SCRIPT_DIR}/_download_meta.py"
|
||||
|
||||
WORKERS="${WORKERS:-1}" # 并行建树进程数（默认串行，保护 API 速率）
|
||||
MIN_DURATION="${MIN_DURATION:-1800}" # 长视频最短时长（秒）
|
||||
MAX_DURATION="${MAX_DURATION:-3600}" # 长视频最长时长（秒）
|
||||
YTDLP_RATE="${YTDLP_RATE:-500K}" # yt-dlp 下载限速（防封）
|
||||
YTDLP_RETRIES="${YTDLP_RETRIES:-5}" # yt-dlp 重试次数
|
||||
|
||||
TIMESTAMP="$(date +%Y%m%d_%H%M%S)"
|
||||
MAIN_LOG="${LOG_DIR}/build_${TIMESTAMP}.log"
|
||||
|
||||
# 颜色输出
|
||||
RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; NC='\033[0m'
|
||||
info() { echo -e "${GREEN}[INFO]${NC} $(date '+%H:%M:%S') $*" | tee -a "${MAIN_LOG}"; }
|
||||
warn() { echo -e "${YELLOW}[WARN]${NC} $(date '+%H:%M:%S') $*" | tee -a "${MAIN_LOG}"; }
|
||||
error() { echo -e "${RED}[ERROR]${NC} $(date '+%H:%M:%S') $*" | tee -a "${MAIN_LOG}"; }
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 1. 创建目录结构
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
info "=== Step 1: 初始化目录结构 ==="
|
||||
mkdir -p "${VIDEO_DIR}" "${META_DIR}" "${TREE_DIR}" "${LOG_DIR}" "${CKPT_DIR}"
|
||||
info "数据目录已就绪: ${DATA_DIR}"
|
||||
info " videos/ → ${VIDEO_DIR}"
|
||||
info " metadata/ → ${META_DIR}"
|
||||
info " trees/ → ${TREE_DIR}"
|
||||
info " logs/ → ${LOG_DIR}"
|
||||
info " checkpoints/ → ${CKPT_DIR}"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 2. 激活 Conda 环境
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
info "=== Step 2: 激活 Conda 环境 (${CONDA_ENV}) ==="
|
||||
|
||||
# 找到 conda 初始化脚本
|
||||
CONDA_BASE="$(conda info --base 2>/dev/null || echo "")"
|
||||
if [[ -z "${CONDA_BASE}" ]]; then
|
||||
error "未找到 conda，请确保 conda 已安装并在 PATH 中"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# shellcheck source=/dev/null
|
||||
source "${CONDA_BASE}/etc/profile.d/conda.sh"
|
||||
conda activate "${CONDA_ENV}"
|
||||
info "已激活环境: $(conda info --envs | grep '*' | awk '{print $1}')"
|
||||
info "Python 路径: $(which python)"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 3. 安装必要工具
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
info "=== Step 3: 安装必要工具 ==="
|
||||
|
||||
pip install --quiet --upgrade yt-dlp datasets
|
||||
info "yt-dlp 版本: $(yt-dlp --version)"
|
||||
python -c "import datasets; print(f'datasets 版本: {datasets.__version__}')"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 4. 下载 VideoMME 元数据，提取长视频列表
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
info "=== Step 4: 下载 VideoMME 元数据 ==="
|
||||
|
||||
LONG_VIDEOS_JSONL="${META_DIR}/long_videos.jsonl"
|
||||
LONG_QA_JSONL="${META_DIR}/long_videos_qa.jsonl"
|
||||
|
||||
if [[ -f "${LONG_VIDEOS_JSONL}" ]]; then
|
||||
EXISTING_COUNT=$(wc -l < "${LONG_VIDEOS_JSONL}")
|
||||
warn "元数据已存在 (${EXISTING_COUNT} 条长视频)，跳过下载。如需重新下载，删除 ${LONG_VIDEOS_JSONL}"
|
||||
else
|
||||
# 配置 HuggingFace 镜像（国内加速，如已能直连可注释掉）
|
||||
export HF_ENDPOINT="${HF_ENDPOINT:-https://hf-mirror.com}"
|
||||
info "HuggingFace 端点: ${HF_ENDPOINT}"
|
||||
|
||||
python "${META_SCRIPT}" \
|
||||
--meta-dir "${META_DIR}" \
|
||||
--min-duration "${MIN_DURATION}" \
|
||||
--max-duration "${MAX_DURATION}" \
|
||||
2>&1 | tee -a "${MAIN_LOG}"
|
||||
fi
|
||||
|
||||
# 确认文件存在
|
||||
if [[ ! -f "${LONG_VIDEOS_JSONL}" ]]; then
|
||||
error "元数据文件不存在，元数据下载失败: ${LONG_VIDEOS_JSONL}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
TOTAL_VIDEOS=$(wc -l < "${LONG_VIDEOS_JSONL}")
|
||||
info "长视频总数: ${TOTAL_VIDEOS}"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 5. 批量下载长视频（yt-dlp，断点续传）
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
info "=== Step 5: 批量下载长视频 ==="
|
||||
info "下载目录: ${VIDEO_DIR}"
|
||||
info "限速: ${YTDLP_RATE}，重试次数: ${YTDLP_RETRIES}"
|
||||
|
||||
DOWNLOAD_LOG="${LOG_DIR}/download_${TIMESTAMP}.log"
|
||||
FAILED_DOWNLOADS="${LOG_DIR}/failed_downloads_${TIMESTAMP}.txt"
|
||||
DOWNLOAD_COUNT=0
|
||||
SKIP_COUNT=0
|
||||
FAIL_COUNT=0
|
||||
|
||||
while IFS= read -r line; do
|
||||
VIDEO_ID=$(echo "${line}" | python -c "import sys,json; d=json.load(sys.stdin); print(d.get('youtube_id') or d.get('video_id',''))")
|
||||
URL=$(echo "${line}" | python -c "import sys,json; d=json.load(sys.stdin); print(d.get('url',''))")
|
||||
|
||||
if [[ -z "${VIDEO_ID}" || -z "${URL}" ]]; then
|
||||
warn "跳过无效记录: ${line:0:80}"
|
||||
continue
|
||||
fi
|
||||
|
||||
# 检查是否已下载（任意格式均算）
|
||||
EXISTING_FILE=$(find "${VIDEO_DIR}" -name "${VIDEO_ID}.*" -type f 2>/dev/null | head -1)
|
||||
if [[ -n "${EXISTING_FILE}" ]]; then
|
||||
SKIP_COUNT=$((SKIP_COUNT + 1))
|
||||
continue
|
||||
fi
|
||||
|
||||
info "[下载 ${DOWNLOAD_COUNT}/${TOTAL_VIDEOS}] ${VIDEO_ID}"
|
||||
|
||||
# yt-dlp 下载：优先 mp4，最高 720p（节省空间），单文件断点续传
|
||||
if yt-dlp \
|
||||
--output "${VIDEO_DIR}/%(id)s.%(ext)s" \
|
||||
--format "bestvideo[ext=mp4][height<=720]+bestaudio[ext=m4a]/best[ext=mp4][height<=720]/best" \
|
||||
--merge-output-format mp4 \
|
||||
--retries "${YTDLP_RETRIES}" \
|
||||
--rate-limit "${YTDLP_RATE}" \
|
||||
--no-playlist \
|
||||
--continue \
|
||||
--no-overwrites \
|
||||
--write-info-json \
|
||||
--quiet \
|
||||
"${URL}" \
|
||||
>> "${DOWNLOAD_LOG}" 2>&1; then
|
||||
DOWNLOAD_COUNT=$((DOWNLOAD_COUNT + 1))
|
||||
info " ✓ 下载成功: ${VIDEO_ID}"
|
||||
else
|
||||
FAIL_COUNT=$((FAIL_COUNT + 1))
|
||||
warn " ✗ 下载失败: ${VIDEO_ID} (${URL})"
|
||||
echo "${VIDEO_ID} ${URL}" >> "${FAILED_DOWNLOADS}"
|
||||
fi
|
||||
|
||||
done < "${LONG_VIDEOS_JSONL}"
|
||||
|
||||
info "下载汇总: 新下载=${DOWNLOAD_COUNT}, 跳过=${SKIP_COUNT}, 失败=${FAIL_COUNT}"
|
||||
if [[ ${FAIL_COUNT} -gt 0 ]]; then
|
||||
warn "失败列表已保存至: ${FAILED_DOWNLOADS}"
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 6. 批量建树（main.py index，跳过缓存命中）
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
info "=== Step 6: 批量建树 ==="
|
||||
info "项目根目录: ${PROJECT_ROOT}"
|
||||
info "配置文件: ${CONFIG_YAML}"
|
||||
info "并行进程数: ${WORKERS}"
|
||||
|
||||
BUILD_LOG="${LOG_DIR}/build_trees_${TIMESTAMP}.log"
|
||||
FAILED_BUILDS="${LOG_DIR}/failed_builds_${TIMESTAMP}.txt"
|
||||
BUILD_COUNT=0
|
||||
BUILD_SKIP=0
|
||||
BUILD_FAIL=0
|
||||
|
||||
# 构建函数（单个视频）
|
||||
build_one_video() {
|
||||
local video_path="$1"
|
||||
local video_stem
|
||||
video_stem="$(basename "${video_path%.*}")"
|
||||
local cache_file="${TREE_DIR}/${video_stem}_video.pkl"
|
||||
|
||||
# 缓存命中则跳过（pipeline.py 内部也会检查，此处提前判断减少日志噪声）
|
||||
if [[ -f "${cache_file}" ]]; then
|
||||
echo "[SKIP] ${video_stem}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo "[BUILD] ${video_stem} ← ${video_path}"
|
||||
if conda run -n "${CONDA_ENV}" python "${PROJECT_ROOT}/main.py" \
|
||||
index \
|
||||
--source "${video_path}" \
|
||||
--modality video \
|
||||
--config "${CONFIG_YAML}" \
|
||||
--env "${ENV_FILE}" \
|
||||
>> "${BUILD_LOG}" 2>&1; then
|
||||
echo "[OK] ${video_stem}"
|
||||
return 0
|
||||
else
|
||||
echo "[FAIL] ${video_stem}"
|
||||
echo "${video_path}" >> "${FAILED_BUILDS}"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
export -f build_one_video
|
||||
export CONDA_ENV PROJECT_ROOT CONFIG_YAML ENV_FILE TREE_DIR BUILD_LOG FAILED_BUILDS
|
||||
|
||||
if [[ "${WORKERS}" -gt 1 ]]; then
|
||||
# 并行模式：使用 GNU parallel
|
||||
if ! command -v parallel &> /dev/null; then
|
||||
warn "未找到 GNU parallel，降级为串行模式"
|
||||
WORKERS=1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "${WORKERS}" -gt 1 ]]; then
|
||||
info "并行建树 (jobs=${WORKERS})..."
|
||||
find "${VIDEO_DIR}" -type f \( -name "*.mp4" -o -name "*.avi" -o -name "*.mkv" -o -name "*.webm" \) \
|
||||
| parallel -j "${WORKERS}" --bar build_one_video {} \
|
||||
2>&1 | tee -a "${MAIN_LOG}"
|
||||
else
|
||||
# 串行模式
|
||||
while IFS= read -r -d '' video_path; do
|
||||
video_stem="$(basename "${video_path%.*}")"
|
||||
cache_file="${TREE_DIR}/${video_stem}_video.pkl"
|
||||
|
||||
if [[ -f "${cache_file}" ]]; then
|
||||
BUILD_SKIP=$((BUILD_SKIP + 1))
|
||||
continue
|
||||
fi
|
||||
|
||||
info "[建树 $((BUILD_COUNT + BUILD_SKIP + 1))/${TOTAL_VIDEOS}] ${video_stem}"
|
||||
if conda run -n "${CONDA_ENV}" python "${PROJECT_ROOT}/main.py" \
|
||||
index \
|
||||
--source "${video_path}" \
|
||||
--modality video \
|
||||
--config "${CONFIG_YAML}" \
|
||||
--env "${ENV_FILE}" \
|
||||
>> "${BUILD_LOG}" 2>&1; then
|
||||
BUILD_COUNT=$((BUILD_COUNT + 1))
|
||||
info " ✓ 建树成功: ${video_stem}"
|
||||
else
|
||||
BUILD_FAIL=$((BUILD_FAIL + 1))
|
||||
warn " ✗ 建树失败: ${video_stem}"
|
||||
echo "${video_path}" >> "${FAILED_BUILDS}"
|
||||
fi
|
||||
done < <(find "${VIDEO_DIR}" -type f \( -name "*.mp4" -o -name "*.avi" -o -name "*.mkv" -o -name "*.webm" \) -print0)
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 7. 最终汇总
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
info "=== Step 7: 汇总 ==="
|
||||
|
||||
TREE_COUNT=$(find "${TREE_DIR}" -name "*_video.pkl" -type f 2>/dev/null | wc -l)
|
||||
VIDEO_COUNT=$(find "${VIDEO_DIR}" -type f \( -name "*.mp4" -o -name "*.avi" -o -name "*.mkv" -o -name "*.webm" \) 2>/dev/null | wc -l)
|
||||
|
||||
info "======================================"
|
||||
info " 长视频元数据数量: ${TOTAL_VIDEOS}"
|
||||
info " 已下载视频数量: ${VIDEO_COUNT}"
|
||||
info " 已完成树索引数量: ${TREE_COUNT}"
|
||||
info " 本次新建树: ${BUILD_COUNT}"
|
||||
info " 跳过（缓存命中）: ${BUILD_SKIP}"
|
||||
info " 建树失败: ${BUILD_FAIL}"
|
||||
info "======================================"
|
||||
info "主日志: ${MAIN_LOG}"
|
||||
info "下载日志: ${DOWNLOAD_LOG}"
|
||||
info "建树日志: ${BUILD_LOG}"
|
||||
|
||||
if [[ ${BUILD_FAIL} -gt 0 ]]; then
|
||||
warn "有 ${BUILD_FAIL} 个视频建树失败，详见: ${FAILED_BUILDS}"
|
||||
warn "可重新运行脚本以续建失败项（自动跳过已缓存）"
|
||||
fi
|
||||
|
||||
if [[ "${TREE_COUNT}" -ge "${TOTAL_VIDEOS}" ]]; then
|
||||
info "✅ 所有长视频树索引已构建完成！"
|
||||
else
|
||||
warn "⚠ 树索引尚未全部完成 (${TREE_COUNT}/${TOTAL_VIDEOS})，可重新运行以续建"
|
||||
fi
|
||||
|
||||
info "脚本完成。"
|
||||
Executable
+95
@@ -0,0 +1,95 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# download_videos.sh — 批量下载 VideoMME 长视频（MP4 含音轨）
|
||||
#
|
||||
# 用法:
|
||||
# bash scripts/download_videos.sh # 下载全部未完成的视频
|
||||
#
|
||||
# 特性:
|
||||
# - 已存在的 mp4 自动跳过（断点续传）
|
||||
# - bestvideo+bestaudio 合并，保证有音轨
|
||||
# - 并发数: 3（避免被 YouTube 限流）
|
||||
# - 失败记录写入 logs/failed_downloads.txt
|
||||
# =============================================================================
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
|
||||
JSONL="${PROJECT_ROOT}/data/videomme/metadata/long_videos.jsonl"
|
||||
VIDEO_DIR="${PROJECT_ROOT}/data/videomme/videos"
|
||||
LOG_DIR="${PROJECT_ROOT}/data/videomme/logs"
|
||||
LOG="${LOG_DIR}/download_$(date +%Y%m%d_%H%M%S).log"
|
||||
FAILED_FILE="${LOG_DIR}/failed_downloads.txt"
|
||||
CONCURRENT=3
|
||||
|
||||
mkdir -p "${VIDEO_DIR}" "${LOG_DIR}"
|
||||
|
||||
# 激活 conda 环境（yt-dlp、ffmpeg 在此环境中）
|
||||
CONDA_BASE="$(conda info --base 2>/dev/null || echo "${HOME}/miniconda3")"
|
||||
# shellcheck source=/dev/null
|
||||
source "${CONDA_BASE}/etc/profile.d/conda.sh"
|
||||
conda activate Video-Tree-TRM
|
||||
|
||||
echo "[$(date)] ===== 批量下载开始 =====" | tee "${LOG}"
|
||||
echo "[$(date)] 视频目录: ${VIDEO_DIR}" | tee -a "${LOG}"
|
||||
|
||||
# 读取所有 youtube_id，写入临时文件供 xargs 读取
|
||||
TMP_IDS=$(mktemp)
|
||||
python3 -c "
|
||||
import json
|
||||
with open('${JSONL}') as f:
|
||||
for line in f:
|
||||
d = json.loads(line.strip())
|
||||
print(d['youtube_id'])
|
||||
" > "${TMP_IDS}"
|
||||
|
||||
TOTAL=$(wc -l < "${TMP_IDS}")
|
||||
echo "[$(date)] 总计: ${TOTAL} 个视频，并发数: ${CONCURRENT}" | tee -a "${LOG}"
|
||||
|
||||
# 3 2.5Mb/s并发下载：每个 youtube_id 单独调用 yt-dlp
|
||||
# 注意：直接在 xargs 中内联参数，不依赖 bash 数组导出
|
||||
xargs -P "${CONCURRENT}" -I {} bash -c '
|
||||
VIDEO_DIR="'"${VIDEO_DIR}"'"
|
||||
LOG="'"${LOG}"'"
|
||||
FAILED_FILE="'"${FAILED_FILE}"'"
|
||||
YID="{}"
|
||||
OUT="${VIDEO_DIR}/${YID}.mp4"
|
||||
|
||||
if [[ -f "${OUT}" && -s "${OUT}" ]]; then
|
||||
echo "[$(date)] [SKIP] ${YID}" >> "${LOG}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
URL="https://www.youtube.com/watch?v=${YID}"
|
||||
echo "[$(date)] [START] ${YID}" >> "${LOG}"
|
||||
|
||||
if yt-dlp \
|
||||
--format "bestvideo[vcodec^=avc][ext=mp4]+bestaudio[ext=m4a]/bestvideo[ext=mp4]+bestaudio[ext=m4a]/bestvideo+bestaudio/best[ext=mp4]/best" \
|
||||
--merge-output-format mp4 \
|
||||
--output "${OUT}" \
|
||||
--no-playlist \
|
||||
--retries 5 \
|
||||
--fragment-retries 5 \
|
||||
--socket-timeout 60 \
|
||||
--no-warnings \
|
||||
--rate-limit 833K \
|
||||
"${URL}" >> "${LOG}" 2>&1; then
|
||||
echo "[$(date)] [OK] ${YID} size=$(du -sh "${OUT}" 2>/dev/null | cut -f1)" >> "${LOG}"
|
||||
else
|
||||
echo "[$(date)] [FAIL] ${YID}" >> "${LOG}"
|
||||
echo "${YID}" >> "${FAILED_FILE}"
|
||||
fi
|
||||
' < "${TMP_IDS}"
|
||||
|
||||
rm -f "${TMP_IDS}"
|
||||
|
||||
TOTAL_MP4=$(find "${VIDEO_DIR}" -name "*.mp4" -size +0c | wc -l)
|
||||
FAILED_COUNT=$(wc -l < "${FAILED_FILE}" 2>/dev/null || echo 0)
|
||||
|
||||
echo "" | tee -a "${LOG}"
|
||||
echo "[$(date)] ===== 汇总 =====" | tee -a "${LOG}"
|
||||
echo "[$(date)] 目录共 MP4: ${TOTAL_MP4} / ${TOTAL}" | tee -a "${LOG}"
|
||||
[[ "${FAILED_COUNT}" -gt 0 ]] && echo "[$(date)] 失败数: ${FAILED_COUNT}，列表: ${FAILED_FILE}" | tee -a "${LOG}"
|
||||
echo "[$(date)] 日志: ${LOG}" | tee -a "${LOG}"
|
||||
Reference in New Issue
Block a user