redup: skip conclusive hard/ref; closes #1652

copyparty/fsutil.py

@@ -10,7 +10,7 @@ import time
 from .__init__ import ANYWIN, FREEBSD, MACOS, UNIX
 from .authsrv import AXS, VFS, AuthSrv
 from .bos import bos
-from .util import chkcmd, json_hesc, min_ex, undot
+from .util import chkcmd, fsenc, json_hesc, min_ex, runcmd, undot
 
 if True:  # pylint: disable=using-constant-test
     from typing import Optional, Union
@@ -18,6 +18,11 @@ if True:  # pylint: disable=using-constant-test
     from .util import RootLogger, undot
 
 
+PTN_FILEFRAG = re.compile(
+    r"^\s*[0-9]+:\s*[0-9]+\.\.\s*[0-9]+:\s*([0-9]+)\.\.\s*([0-9]+):\s*[0-9]+:\s*(.*)$"
+)
+
+
 class Fstab(object):
     def __init__(self, log: "RootLogger", args: argparse.Namespace, verbose: bool):
         self.log_func = log
@@ -264,3 +269,17 @@ def ramdisk_chk(asrv: AuthSrv) -> None:
         vol.flags["fsnt"] = vol.js_ls["fsnt"] = htm["fsnt"] = fs
         vol.js_htm = json_hesc(json.dumps(htm))
         # md_htm is dontcare; only relevant for up2k request body
+
+
+def filefrag(ap: str) -> list[str]:
+    rc, so, se = runcmd([b"filefrag", b"-v", fsenc(ap)], timeout=10)
+    ptn = PTN_FILEFRAG
+    ret = []
+    for ln in so.split("\n"):
+        m = ptn.match(ln)
+        if not m:
+            continue
+        a, b, s = m.groups()
+        if "shared" in s.strip().split(","):
+            ret.append((a, b))
+    return ret

copyparty/up2k.py

@@ -20,7 +20,7 @@ from .__init__ import ANYWIN, PY2, TYPE_CHECKING, UNIX, WINDOWS, E
 from .authsrv import LEELOO_DALLAS, REDUP_E2, SEESLOG, VFS, AuthSrv
 from .bos import bos
 from .cfg import vf_bmap, vf_cmap, vf_vmap
-from .fsutil import Fstab
+from .fsutil import Fstab, filefrag
 from .mtag import MParser, MTag
 from .util import (
     E_FS_CRIT,
@@ -2181,10 +2181,10 @@ class Up2k(object):
             for w, rd, fn in cur.execute("select w, rd, fn from up"):
                 w16 = w[:16]
                 for v2, c2 in curs:
-                    hit = c2.execute(q2, (w16, w)).fetchone()
-                    if not hit:
+                    hits = c2.execute(q2, (w16, w)).fetchall()
+                    if not hits:
                         continue
-                    rd2, fn2 = hit
+                    rd2, fn2 = hits[0]
                     if fn == fn2 and rd == rd2 and vol is v2:
                         continue
                     apt = ""
@@ -2196,11 +2196,22 @@ class Up2k(object):
                         st1 = bos.lstat(fp1)
                         if fp1 == fp2 or stat.S_ISLNK(st1.st_mode):
                             continue  # self(?), or redup failed during walk
+                        if to == "hard" and st1.st_nlink >= len(hits):
+                            continue
                         ap1 = absreal(fp1)
                         ap2 = absreal(fp2)
                         st2 = bos.lstat(ap2)
                         if stat.S_ISLNK(st2.st_mode):
                             continue  # dead hit
+                        if to == "ref" and ap1 != ap2:
+                            try:
+                                frag1 = filefrag(ap1)
+                                if frag1:
+                                    frag2 = filefrag(ap2)
+                                    if frag1 == frag2:
+                                        continue
+                            except Exception as ex:
+                                self.log("redup: filefrag failed (OK); %r" % (ex,))
                         for ap in (ap1,) if ap1 == ap2 else (ap1, ap2):
                             self.log("redup: integrity-checking %r" % (ap,))
                             zsl, st = self._hashlist_from_file(ap)