From 080935788b1730320696f102412e1c1964007fd4 Mon Sep 17 00:00:00 2001 From: ed Date: Wed, 30 Sep 2026 00:05:02 +0000 Subject: [PATCH] redup: skip conclusive hard/ref; closes #1652 --- copyparty/fsutil.py | 21 ++++++++++++++++++++- copyparty/up2k.py | 19 +++++++++++++++---- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/copyparty/fsutil.py b/copyparty/fsutil.py index f911a3f8d..f843bd3c0 100644 --- a/copyparty/fsutil.py +++ b/copyparty/fsutil.py @@ -10,7 +10,7 @@ import time from .__init__ import ANYWIN, FREEBSD, MACOS, UNIX from .authsrv import AXS, VFS, AuthSrv from .bos import bos -from .util import chkcmd, json_hesc, min_ex, undot +from .util import chkcmd, fsenc, json_hesc, min_ex, runcmd, undot if True: # pylint: disable=using-constant-test from typing import Optional, Union @@ -18,6 +18,11 @@ if True: # pylint: disable=using-constant-test from .util import RootLogger, undot +PTN_FILEFRAG = re.compile( + r"^\s*[0-9]+:\s*[0-9]+\.\.\s*[0-9]+:\s*([0-9]+)\.\.\s*([0-9]+):\s*[0-9]+:\s*(.*)$" +) + + class Fstab(object): def __init__(self, log: "RootLogger", args: argparse.Namespace, verbose: bool): self.log_func = log @@ -264,3 +269,17 @@ def ramdisk_chk(asrv: AuthSrv) -> None: vol.flags["fsnt"] = vol.js_ls["fsnt"] = htm["fsnt"] = fs vol.js_htm = json_hesc(json.dumps(htm)) # md_htm is dontcare; only relevant for up2k request body + + +def filefrag(ap: str) -> list[str]: + rc, so, se = runcmd([b"filefrag", b"-v", fsenc(ap)], timeout=10) + ptn = PTN_FILEFRAG + ret = [] + for ln in so.split("\n"): + m = ptn.match(ln) + if not m: + continue + a, b, s = m.groups() + if "shared" in s.strip().split(","): + ret.append((a, b)) + return ret diff --git a/copyparty/up2k.py b/copyparty/up2k.py index dc113975e..c8e658550 100644 --- a/copyparty/up2k.py +++ b/copyparty/up2k.py @@ -20,7 +20,7 @@ from .__init__ import ANYWIN, PY2, TYPE_CHECKING, UNIX, WINDOWS, E from .authsrv import LEELOO_DALLAS, REDUP_E2, SEESLOG, VFS, AuthSrv from .bos import bos from .cfg import vf_bmap, vf_cmap, vf_vmap -from .fsutil import Fstab +from .fsutil import Fstab, filefrag from .mtag import MParser, MTag from .util import ( E_FS_CRIT, @@ -2181,10 +2181,10 @@ class Up2k(object): for w, rd, fn in cur.execute("select w, rd, fn from up"): w16 = w[:16] for v2, c2 in curs: - hit = c2.execute(q2, (w16, w)).fetchone() - if not hit: + hits = c2.execute(q2, (w16, w)).fetchall() + if not hits: continue - rd2, fn2 = hit + rd2, fn2 = hits[0] if fn == fn2 and rd == rd2 and vol is v2: continue apt = "" @@ -2196,11 +2196,22 @@ class Up2k(object): st1 = bos.lstat(fp1) if fp1 == fp2 or stat.S_ISLNK(st1.st_mode): continue # self(?), or redup failed during walk + if to == "hard" and st1.st_nlink >= len(hits): + continue ap1 = absreal(fp1) ap2 = absreal(fp2) st2 = bos.lstat(ap2) if stat.S_ISLNK(st2.st_mode): continue # dead hit + if to == "ref" and ap1 != ap2: + try: + frag1 = filefrag(ap1) + if frag1: + frag2 = filefrag(ap2) + if frag1 == frag2: + continue + except Exception as ex: + self.log("redup: filefrag failed (OK); %r" % (ex,)) for ap in (ap1,) if ap1 == ap2 else (ap1, ap2): self.log("redup: integrity-checking %r" % (ap,)) zsl, st = self._hashlist_from_file(ap)