feat: repair 重传后自动复核 (前置校验等级验证重传文件)
This commit is contained in:
1 parent
4defbf8b1e
commit
30419c9aae
3 files changed
+100
No files matched your search
@@ -126,6 +126,8 @@ python3 transfer.py --mode cli --check-only --verify-level slow --repair /src/ /
|
|||||||
> `--repair`: 校验发现问题文件后提示 [y/N],确认后用 rsync 重传问题文件。
|
> `--repair`: 校验发现问题文件后提示 [y/N],确认后用 rsync 重传问题文件。
|
||||||
> 重传前**先删除目标问题文件**——避免 rsync 按 size+mtime 判定相同而跳过
|
> 重传前**先删除目标问题文件**——避免 rsync 按 size+mtime 判定相同而跳过
|
||||||
> (内容损坏但大小/时间戳一致的场景,增量检测有盲区)。
|
> (内容损坏但大小/时间戳一致的场景,增量检测有盲区)。
|
||||||
|
> 重传完成后**自动复核**:用前置校验等级(`--verify-level`)对重传文件
|
||||||
|
> 再校验一遍,确认修复成功(日志记录 `复核通过 X 个 / 失败 Y 个`)。
|
||||||
|
|
||||||
> rsync 原子写(临时文件+rename)保证掉卡不会产生半截损坏的正式文件,
|
> rsync 原子写(临时文件+rename)保证掉卡不会产生半截损坏的正式文件,
|
||||||
> 残留临时文件由补传后的 `--verify` 流程发现处理。
|
> 残留临时文件由补传后的 `--verify` 流程发现处理。
|
||||||
|
|||||||
@@ -593,3 +593,41 @@ def test_build_repair_cmd(tmp_path):
|
|||||||
assert "--mkpath" in joined
|
assert "--mkpath" in joined
|
||||||
assert "--bwlimit=30000" in joined
|
assert "--bwlimit=30000" in joined
|
||||||
assert "src/" in joined and "dst/" in joined
|
assert "src/" in joined and "dst/" in joined
|
||||||
|
|
||||||
|
# ---------- 重传后复核 ----------
|
||||||
|
|
||||||
|
def test_verify_files_list_all_ok(tmp_path):
|
||||||
|
src = tmp_path / "s"; dst = tmp_path / "d"
|
||||||
|
src.mkdir(); dst.mkdir()
|
||||||
|
for f in ("a.flac", "b.lrc"):
|
||||||
|
(src / f).write_bytes(b"data" * 100)
|
||||||
|
(dst / f).write_bytes(b"data" * 100)
|
||||||
|
ok, fail = transfer.verify_files_list(str(src), str(dst), ["a.flac", "b.lrc"])
|
||||||
|
assert set(ok) == {"a.flac", "b.lrc"} and fail == []
|
||||||
|
|
||||||
|
def test_verify_files_list_detect_fail(tmp_path):
|
||||||
|
src = tmp_path / "s"; dst = tmp_path / "d"
|
||||||
|
src.mkdir(); dst.mkdir()
|
||||||
|
(src / "a.flac").write_bytes(b"good content")
|
||||||
|
(dst / "a.flac").write_bytes(b"bad content!!")
|
||||||
|
ok, fail = transfer.verify_files_list(str(src), str(dst), ["a.flac"])
|
||||||
|
assert fail == ["a.flac"] and ok == []
|
||||||
|
|
||||||
|
def test_verify_files_list_ultrafast_size_only(tmp_path):
|
||||||
|
"""> 同大小不同内容: ultrafast 判通过, slow 判失败"""
|
||||||
|
src = tmp_path / "s"; dst = tmp_path / "d"
|
||||||
|
src.mkdir(); dst.mkdir()
|
||||||
|
data = b"x" * 1000
|
||||||
|
(src / "a.flac").write_bytes(data)
|
||||||
|
(dst / "a.flac").write_bytes(b"y" * 1000)
|
||||||
|
ok, _ = transfer.verify_files_list(str(src), str(dst), ["a.flac"], level="ultrafast")
|
||||||
|
assert ok == ["a.flac"] # 只查大小
|
||||||
|
ok2, fail2 = transfer.verify_files_list(str(src), str(dst), ["a.flac"], level="slow")
|
||||||
|
assert fail2 == ["a.flac"] # md5 查出
|
||||||
|
|
||||||
|
def test_verify_files_list_missing_target(tmp_path):
|
||||||
|
src = tmp_path / "s"; dst = tmp_path / "d"
|
||||||
|
src.mkdir(); dst.mkdir()
|
||||||
|
(src / "a.flac").write_bytes(b"x")
|
||||||
|
ok, fail = transfer.verify_files_list(str(src), str(dst), ["a.flac"])
|
||||||
|
assert fail == ["a.flac"]
|
||||||
+60
@@ -209,6 +209,40 @@ def build_repair_cmd(src: str, dst: str, list_file: str,
|
|||||||
return cmd, list_file
|
return cmd, list_file
|
||||||
|
|
||||||
|
|
||||||
|
def verify_files_list(src_dir: str, dst_dir: str, rels: list[str],
|
||||||
|
level: str = "slow",
|
||||||
|
on_progress=None) -> tuple[list[str], list[str]]:
|
||||||
|
"""复核指定文件列表: 按校验等级验证源/目标一致性
|
||||||
|
|
||||||
|
- ultrafast: 仅大小
|
||||||
|
- fast: 大小+mtime
|
||||||
|
- normal/slow/full: md5 内容校验
|
||||||
|
返回 (通过列表, 失败列表); on_progress(i, total) 进度回调
|
||||||
|
"""
|
||||||
|
ok: list[str] = []
|
||||||
|
fail: list[str] = []
|
||||||
|
total = len(rels)
|
||||||
|
for i, rel in enumerate(rels, 1):
|
||||||
|
sp = os.path.join(src_dir, rel)
|
||||||
|
tp = os.path.join(dst_dir, rel)
|
||||||
|
if not os.path.isfile(sp):
|
||||||
|
continue
|
||||||
|
if not os.path.isfile(tp):
|
||||||
|
fail.append(rel)
|
||||||
|
elif level == "ultrafast":
|
||||||
|
(ok if os.path.getsize(sp) == os.path.getsize(tp) else fail).append(rel)
|
||||||
|
elif level == "fast":
|
||||||
|
sst, tst = os.stat(sp), os.stat(tp)
|
||||||
|
good = (sst.st_size == tst.st_size
|
||||||
|
and int(sst.st_mtime) == int(tst.st_mtime))
|
||||||
|
(ok if good else fail).append(rel)
|
||||||
|
else:
|
||||||
|
(ok if files_identical(sp, tp) else fail).append(rel)
|
||||||
|
if on_progress:
|
||||||
|
on_progress(i, total)
|
||||||
|
return ok, fail
|
||||||
|
|
||||||
|
|
||||||
def read_proc_io(pid: int) -> int:
|
def read_proc_io(pid: int) -> int:
|
||||||
"""读取进程累计读字节数 (/proc/pid/io read_bytes)"""
|
"""读取进程累计读字节数 (/proc/pid/io read_bytes)"""
|
||||||
try:
|
try:
|
||||||
@@ -954,6 +988,32 @@ class TaskRunner:
|
|||||||
self.on_log("[green]✔ 问题文件已重传[/]")
|
self.on_log("[green]✔ 问题文件已重传[/]")
|
||||||
else:
|
else:
|
||||||
self.on_log(f"[red]✘ 重传失败 rc={rc}: {errs.strip()[:200]}[/]")
|
self.on_log(f"[red]✘ 重传失败 rc={rc}: {errs.strip()[:200]}[/]")
|
||||||
|
# 重传后复核: 用前置校验等级再验一遍重传的文件
|
||||||
|
if rc == 0 and issues:
|
||||||
|
self._recheck_repaired(issues)
|
||||||
|
|
||||||
|
def _recheck_repaired(self, issues: list[str]) -> None:
|
||||||
|
"""重传后用前置校验等级复核问题文件"""
|
||||||
|
src_dir = self.cfg.source.rstrip("/")
|
||||||
|
dst_dir = self.cfg.dest.rstrip("/")
|
||||||
|
self.lm.write(f"repair: 复核 {len(issues)} 个重传文件 "
|
||||||
|
f"(level={self.cfg.verify_level})")
|
||||||
|
if self.on_log:
|
||||||
|
self.on_log(f"[cyan]复核重传文件 ({self.cfg.verify_level})...[/]")
|
||||||
|
|
||||||
|
def on_progress(i: int, total: int) -> None:
|
||||||
|
self._emit_verify_progress(i, total)
|
||||||
|
|
||||||
|
ok, fail = verify_files_list(src_dir, dst_dir, issues,
|
||||||
|
self.cfg.verify_level,
|
||||||
|
on_progress=on_progress)
|
||||||
|
self.lm.write(f"repair: 复核完成 通过={len(ok)} 失败={len(fail)}")
|
||||||
|
if self.on_log:
|
||||||
|
if not fail:
|
||||||
|
self.on_log(f"[green]✔ 复核通过: {len(ok)} 个重传文件完整[/]")
|
||||||
|
else:
|
||||||
|
self.on_log(f"[red]✘ 复核发现 {len(fail)} 个文件仍未修复: "
|
||||||
|
f"{', '.join(os.path.basename(f) for f in fail[:5])}[/]")
|
||||||
|
|
||||||
def _emit_verify_progress(self, i: int, total: int) -> None:
|
def _emit_verify_progress(self, i: int, total: int) -> None:
|
||||||
"""校验进度: 复用 stats 驱动进度条 + 每 5% 打日志"""
|
"""校验进度: 复用 stats 驱动进度条 + 每 5% 打日志"""
|
||||||
|
|||||||
Reference in new issue
Block a user