feat: 校验进度条+单文件日志 (on_file 回调)
This commit is contained in:
1 parent
0f9866a031
commit
0b28d26a18
3 files changed
+55
-4
No files matched your search
@@ -88,6 +88,8 @@ help / clear / quit
|
||||
**智能续传(`--smart-resume`)**: 中断后重跑时,只校验目标**最新 N 个文件**
|
||||
(`--smart-check N`,默认 5;≤100MB 做 md5 全校验,更大只查大小),
|
||||
发现损坏立即删除该文件并从断点续传——避免对已传的 100+GB 全量重扫。
|
||||
校验过程有**进度条**(总进度条复用)+ **单文件日志**(`校验 i/N: 文件名 ✓/✘ 损坏`),
|
||||
N 调大(如 50)也能实时看到校验进展。
|
||||
|
||||
> rsync 原子写(临时文件+rename)保证掉卡不会产生半截损坏的正式文件,
|
||||
> 残留临时文件由补传后的 `--verify` 流程发现处理。
|
||||
|
||||
@@ -428,3 +428,30 @@ def test_parse_smart_resume():
|
||||
assert transfer.parse_args(["--smart-resume", "s", "d"]).smart_resume is True
|
||||
assert transfer.parse_args(["--smart-check", "3", "s", "d"]).smart_check == 3
|
||||
assert transfer.parse_args(["s", "d"]).smart_check == 5
|
||||
|
||||
# ---------- 校验进度回调 ----------
|
||||
|
||||
def test_verify_recent_progress_callback(tmp_path):
|
||||
"""校验每个文件触发回调 (index, total, rel, ok)"""
|
||||
src = tmp_path / "s"; dst = tmp_path / "d"
|
||||
src.mkdir(); dst.mkdir()
|
||||
for i in range(3):
|
||||
(src / f"f{i}.txt").write_text("good" * 100)
|
||||
(dst / f"f{i}.txt").write_text("good" * 100)
|
||||
(dst / "f2.txt").write_text("corrupt") # 损坏一个
|
||||
events = []
|
||||
transfer.verify_recent_files(str(src), str(dst), check_count=3,
|
||||
on_file=lambda i, n, r, ok: events.append((i, n, r, ok)))
|
||||
assert len(events) == 3 # 每个文件都回调
|
||||
assert all(n == 3 for _, n, _, _ in events) # total 一致
|
||||
bad_events = [e for e in events if not e[3]]
|
||||
assert len(bad_events) == 1 and bad_events[0][2] == "f2.txt" # 损坏的被标记
|
||||
assert sum(1 for e in events if e[3]) == 2 # 其余通过
|
||||
|
||||
def test_verify_recent_callback_without_progress(tmp_path):
|
||||
"""不带回调时行为不变 (向后兼容)"""
|
||||
src = tmp_path / "s"; dst = tmp_path / "d"
|
||||
src.mkdir(); dst.mkdir()
|
||||
(src / "a.txt").write_text("x")
|
||||
(dst / "a.txt").write_text("x")
|
||||
assert transfer.verify_recent_files(str(src), str(dst)) == []
|
||||
+26
-4
@@ -97,11 +97,13 @@ def files_identical(path_a: str, path_b: str, chunk: int = 1 << 20) -> bool:
|
||||
|
||||
|
||||
def verify_recent_files(src_dir: str, dst_dir: str, check_count: int = 5,
|
||||
max_size: int = 100 * 1024 * 1024) -> list[str]:
|
||||
max_size: int = 100 * 1024 * 1024,
|
||||
on_file=None) -> list[str]:
|
||||
"""智能续传: 校验目标最新 N 个文件与源一致性, 返回损坏/缺失列表并删除目标坏副本
|
||||
|
||||
- <=max_size 的文件: 大小 + 内容 hash 全校验
|
||||
- >max_size 的文件: 只查大小 (内容由后续 rsync --append 兜底)
|
||||
- on_file(i, total, rel, ok): 每文件进度回调 (i 从 1 开始)
|
||||
- 返回需重传的相对路径列表 (目标坏副本已删除)
|
||||
"""
|
||||
bad: list[str] = []
|
||||
@@ -113,19 +115,28 @@ def verify_recent_files(src_dir: str, dst_dir: str, check_count: int = 5,
|
||||
full = os.path.join(root, name)
|
||||
recent.append((os.path.getmtime(full), os.path.relpath(full, dst_dir)))
|
||||
recent.sort(reverse=True)
|
||||
for _mtime, rel in recent[:check_count]:
|
||||
total = min(check_count, len(recent))
|
||||
for idx, (_mtime, rel) in enumerate(recent[:check_count], 1):
|
||||
sp = os.path.join(src_dir, rel)
|
||||
tp = os.path.join(dst_dir, rel)
|
||||
if not os.path.isfile(sp):
|
||||
continue # 源无此文件(目标多余), 不处理
|
||||
if on_file:
|
||||
on_file(idx, total, rel, True) # 源无此文件, 跳过
|
||||
continue
|
||||
if not os.path.isfile(tp) or os.path.getsize(sp) != os.path.getsize(tp):
|
||||
bad.append(rel)
|
||||
if os.path.exists(tp):
|
||||
os.remove(tp)
|
||||
if on_file:
|
||||
on_file(idx, total, rel, False)
|
||||
continue
|
||||
if os.path.getsize(tp) <= max_size and not files_identical(sp, tp):
|
||||
bad.append(rel)
|
||||
os.remove(tp)
|
||||
if on_file:
|
||||
on_file(idx, total, rel, False)
|
||||
elif on_file:
|
||||
on_file(idx, total, rel, True)
|
||||
return bad
|
||||
|
||||
|
||||
@@ -688,7 +699,18 @@ class TaskRunner:
|
||||
self.lm.write(f"smart-resume: 校验目标最新 {self.cfg.smart_check} 个文件...")
|
||||
if self.on_log:
|
||||
self.on_log(f"[cyan]智能续传: 校验最近 {self.cfg.smart_check} 个文件...[/]")
|
||||
bad = verify_recent_files(src_dir, dst_dir, self.cfg.smart_check)
|
||||
def on_verify_file(idx: int, total: int, rel: str, ok: bool) -> None:
|
||||
# 校验进度: 复用 stats 驱动进度条 + 日志栏每文件输出
|
||||
self.stats.bytes_done = idx
|
||||
self.stats.bytes_total = total
|
||||
self.stats.pct = idx / max(1, total) * 100
|
||||
self._emit_stats()
|
||||
if self.on_log:
|
||||
mark = "✓" if ok else "✘ 损坏"
|
||||
self.on_log(f"校验 {idx}/{total}: {os.path.basename(rel)} {mark}")
|
||||
|
||||
bad = verify_recent_files(src_dir, dst_dir, self.cfg.smart_check,
|
||||
on_file=on_verify_file)
|
||||
if bad:
|
||||
self.lm.write(f"smart-resume: 发现 {len(bad)} 个损坏/缺失, 将重传: {bad}")
|
||||
if self.on_log:
|
||||
|
||||
Reference in new issue
Block a user