add --redup

copyparty/__main__.py

@@ -1437,6 +1437,8 @@ def add_upload(ap):
     ap2.add_argument("--hardlink", action="store_true", help="enable hardlink-based dedup; will fallback on symlinks when that is impossible (across filesystems) (volflag=hardlink)")
     ap2.add_argument("--hardlink-only", action="store_true", help="do not fallback to symlinks when a hardlink cannot be made (volflag=hardlinkonly)")
     ap2.add_argument("--reflink", action="store_true", help="enable reflink-based dedup; will fallback on full copies when that is impossible (non-CoW filesystem) (volflag=reflink)")
+    ap2.add_argument("--redup", metavar="A[,B]=T", help="convert dedup-type for existing files; \033[33mA,B\033[0m is dedup-types to convert from (no/ref/sym/hard), \033[33mT\033[0m is target type; converting from sym/hard is fast-ish, from no/ref is slooow; example: [\033[32msym,hard=ref\033[0m] (volflag=redup)")
+    ap2.add_argument("--redup-dry", action="store_true", help="dry-run; makes \033[33m--redup\033[0m not apply changes (volflag=redup_dry)")
     ap2.add_argument("--no-dupe", action="store_true", help="reject duplicate files during upload; only matches within the same volume (volflag=nodupe)")
     ap2.add_argument("--no-dupe-m", action="store_true", help="also reject dupes when moving a file into another volume (volflag=nodupem)")
     ap2.add_argument("--no-clone", action="store_true", help="do not use existing data on disk to satisfy dupe uploads; reduces server HDD reads in exchange for much more network load (volflag=noclone)")

copyparty/authsrv.py

@@ -107,6 +107,7 @@ SEESLOG = " (see fileserver log for details)"
 SSEELOG = " ({})".format(SEE_LOG)
 BAD_CFG = "invalid config; {}".format(SEE_LOG)
 SBADCFG = " ({})".format(BAD_CFG)
+REDUP_E2 = "WARNING: [/%s] needs 'e2ds' for redup from 'ref' and/or to 'sym'"
 
 PTN_U_GRP = re.compile(r"\$\{u(%[+-][^}]+)\}")
 PTN_G_GRP = re.compile(r"\$\{g(%[+-][^}]+)\}")
@@ -2915,6 +2916,17 @@ class AuthSrv(object):
             if ccs == "y":
                 vol.flags["bcasechk"] = True
 
+        ptn = re.compile(r"^(,no|,sym|,hard|,ref)+=(sym|hard|ref)$")
+        ptn2 = re.compile(r"(no|ref).*=|=sym")
+        for vol in vfs.all_nodes.values():
+            zs = vol.flags.get("redup")
+            if zs and not ptn.match("," + zs):
+                t = "invalid redup for volume /%s: %r"
+                self.log(t % (vol.vpath, zs), 1)
+                errors = True
+            if zs and "e2ds" not in vol.flags and ptn2.search(zs):
+                self.log(REDUP_E2 % (vol.vpath,), 3)
+
         tags = self.args.mtp or []
         tags = [x.split("=")[0] for x in tags]
         tags = [y for x in tags for y in x.split(",")]

copyparty/cfg.py

@@ -66,6 +66,7 @@ def vf_bmap() -> dict[str, str]:
         "og_s_title",
         "opds",
         "rand",
+        "redup_dry",
         "reflink",
         "rm_partial",
         "rmagic",
@@ -146,6 +147,7 @@ def vf_vmap() -> dict[str, str]:
         "put_ck",
         "put_name",
         "readmes",
+        "redup",
         "rotf_tz",
         "mv_retry",
         "rm_retry",
@@ -229,6 +231,8 @@ flagcats = {
         "hardlinkonly": "dedup with hardlink only, never symlink;\nmake a full copy if hardlink is impossible",
         "reflink": "enable reflink-based file deduplication,\nwith fallback on full copy when that is impossible",
         "safededup": "verify on-disk data before using it for dedup",
+        "redup=A,B=T": "convert dedup-type for existing files",
+        "redup_dry": "dry-run mode for redup",
         "noclone": "take dupe data from clients, even if available on HDD",
         "nodupe": "rejects existing files (instead of linking/cloning them)",
         "nodupem": "rejects existing files during moves as well",

copyparty/up2k.py

@@ -17,7 +17,7 @@ from copy import deepcopy
 from queue import Queue
 
 from .__init__ import ANYWIN, PY2, TYPE_CHECKING, UNIX, WINDOWS, E
-from .authsrv import LEELOO_DALLAS, SEESLOG, VFS, AuthSrv
+from .authsrv import LEELOO_DALLAS, REDUP_E2, SEESLOG, VFS, AuthSrv
 from .bos import bos
 from .cfg import vf_bmap, vf_cmap, vf_vmap
 from .fsutil import Fstab
@@ -1000,6 +1000,21 @@ class Up2k(object):
             with self.mutex, self.reg_mutex:
                 self._drop_caches()
 
+        def setstate(vol, n):
+            t = ""
+            if n < 1 and "e2v" in vol.flags:
+                t += "+e2v"
+            if n < 2 and "redup" in vol.flags:
+                t += "+redup"
+            if n < 3 and "e2ts" in vol.flags:
+                t += "+tags"
+
+            if t:
+                t = "online (%s pending)" % (t[1:],)
+            else:
+                t = "online, idle"
+            self.volstate[vol.vpath] = t
+
         for vol in vols:
             if self.stop or gid != self.gid:
                 break
@@ -1016,26 +1031,16 @@ class Up2k(object):
                 if vac:
                     need_vac[vol] = True
 
-            if "e2v" in vol.flags:
+            setstate(vol, 0)
-                t = "online (integrity-check pending)"
-            elif "e2ts" in vol.flags:
-                t = "online (tags pending)"
-            else:
-                t = "online, idle"
-
-            self.volstate[vol.vpath] = t
 
         self._unblock()
 
         # file contents verification
         for vol in vols:
-            if self.stop:
+            if self.stop or "e2v" not in vol.flags:
-                break
-
-            if "e2v" not in vol.flags:
                 continue
 
-            t = "online (verifying integrity)"
+            t = "online (doing e2v)"
             self.volstate[vol.vpath] = t
             self.log("{} [{}]".format(t, vol.realpath))
 
@@ -1044,12 +1049,23 @@ class Up2k(object):
                 self.log("modified {} entries in the db".format(nmod), 3)
                 need_vac[vol] = True
 
-            if "e2ts" in vol.flags:
+            setstate(vol, 1)
-                t = "online (tags pending)"
-            else:
-                t = "online, idle"
 
+        # convert dedup type
+        for vol in vols:
+            if self.stop or not vol.flags.get("redup"):
+                continue
+
+            t = "online (doing redup)"
             self.volstate[vol.vpath] = t
+            self.log("{} [{}]".format(t, vol.realpath))
+
+            try:
+                with self.mutex, self.reg_mutex:
+                    self._redup(vol, vols)
+            except:
+                self.log("redup failed: " + min_ex(), 1)
+            setstate(vol, 2)
 
         # open the rest + do any e2ts(a)
         needed_mutagen = False
@@ -2088,6 +2104,122 @@ class Up2k(object):
 
         return len(rewark) + len(f404)
 
+    def _redup(self, vol: VFS, vols: list[VFS]) -> None:
+        logmsg = "redup: replacing %r with %r"
+        dry = "redup_dry" in vol.flags
+        if dry:
+            logmsg += " #dry"
+
+        zs, to = vol.flags["redup"].split("=")
+        zsl = zs.split(",")
+        cref = "no" in zsl or "ref" in zsl
+        csym = "sym" in zsl
+        chard = "hard" in zsl
+        chard0 = chard
+
+        vf = vol.flags.copy()
+        vf["dedup"] = 1
+        for zs in "hardlink hardlinkonly reflink".split():
+            vf.pop(zs, None)
+
+        if to == "ref":
+            vf["reflink"] = 1
+        elif to == "hard":
+            vf["hardlinkonly"] = vf["hardlink"] = 1
+        elif to == "sym":
+            pass
+        else:
+            raise Exception("target type not implemented: " + to)
+
+        if (csym or chard) and to != "sym":
+            self.log("redup: stage 1 begin")
+            if to == "hard":
+                chard = False
+            for x in vol.walk("", "", [], LEELOO_DALLAS, [[]], 2, True, True, True):
+                vn, _, _, atop, lsf, lsd, lsv = x
+                if vn is not vol:
+                    lsd[:] = []
+                    lsv.clear()
+                    continue
+                for fn, st in lsf:
+                    if (csym and stat.S_ISLNK(st.st_mode)) or (
+                        chard and st.st_nlink > 1
+                    ):
+                        ap = os.path.join(atop, fn)
+                        self.log(logmsg % (ap, absreal(ap)))
+                        if dry:
+                            continue
+                        ap2 = tempfile.NamedTemporaryFile(
+                            prefix="r,", dir=atop, delete=False
+                        ).name
+                        self._symlink(ap, ap2, vf, True, True, st.st_mtime)
+                        wunlink(self.log, ap, vf)
+                        bos.rename(ap2, ap)
+
+        if cref or to == "sym":
+            self.log("redup: stage 2 begin")
+            if "e2ds" not in vol.flags:
+                self.log(REDUP_E2 % (vol.vpath,), 3)
+            cur = self.cur[vol.realpath]
+            curs = [(vol, cur.connection.cursor())]
+            if to != "sym":
+                for v2 in vols:
+                    if vol is not v2 and v2.realpath in self.cur:
+                        curs += [(v2, self.cur[v2.realpath])]
+            nrem = cur.execute("select count(w) from up").fetchone()[0]
+            self.log("redup [/%s]: %d files left" % (vol.vpath, nrem))
+            q2 = "select rd, fn from up where substr(w,1,16) = ? and w=?"
+            for w, rd, fn in cur.execute("select w, rd, fn from up"):
+                w16 = w[:16]
+                for v2, c2 in curs:
+                    hit = c2.execute(q2, (w16, w)).fetchone()
+                    if not hit:
+                        continue
+                    rd2, fn2 = hit
+                    if fn == fn2 and rd == rd2 and vol is v2:
+                        continue
+                    try:
+                        apt = ""
+                        rd, fn = s3dec(rd, fn)
+                        rd2, fn2 = s3dec(rd2, fn2)
+                        fp1 = os.path.join(vol.realpath, rd, fn)
+                        fp2 = os.path.join(v2.realpath, rd2, fn2)
+                        st1 = bos.lstat(fp1)
+                        if fp1 == fp2 or stat.S_ISLNK(st1.st_mode):
+                            continue  # self(?), or redup failed during walk
+                        ap1 = absreal(fp1)
+                        ap2 = absreal(fp2)
+                        st2 = bos.lstat(ap2)
+                        if stat.S_ISLNK(st2.st_mode):
+                            continue  # dead hit
+                        for ap in (ap1,) if ap1 == ap2 else (ap1, ap2):
+                            self.log("redup: integrity-checking %r" % (ap,))
+                            zsl, st = self._hashlist_from_file(ap)
+                            w2 = up2k_wark_from_hashlist(self.salt, st.st_size, zsl)
+                            if w2 != w:
+                                t = "db desync:\n%s %r\n%s db"
+                                raise Exception(t % (w2, ap, w))
+
+                        self.log(logmsg % (ap1, ap2))
+                        if dry:
+                            continue
+                        apt = tempfile.NamedTemporaryFile(
+                            prefix="r,", dir=os.path.dirname(fp1), delete=False
+                        ).name
+                        self._symlink(ap2, apt, vf, True, True, st1.st_mtime)
+                        zsl, st = self._hashlist_from_file(apt)
+                        w2 = up2k_wark_from_hashlist(self.salt, st.st_size, zsl)
+                        if w != w2:
+                            raise Exception("bad checksum after copy?!")
+                        wunlink(self.log, ap1, vf)
+                        bos.rename(apt, ap1)
+                        break
+                    except Exception as ex:
+                        self.log("redup: skipping match due to " + str(ex))
+                        if apt and os.path.exists(apt):
+                            os.unlink(apt)
+                        continue
+
     def _build_tags_index(self, vol: VFS) -> tuple[int, int, bool]:
         ptop = vol.realpath
         with self.mutex, self.reg_mutex:

copyparty/util.py

@@ -450,6 +450,7 @@ IMPLICATIONS = [
     ["no_logue", "no_logues"],  # user-typo
     ["nologues", "no_logues"],  # user-typo
     ["nologue", "no_logues"],  # user-typo
+    ["nw", "redup_dry"],
     ["plainlogue", "plainlogues"],  # user-typo
     ["plainreadmes", "plainreadme"],  # user-typo
     ["noscript", "plainlogues"],

scripts/run-tests.sh

@@ -70,6 +70,9 @@ done
 [ "$1" ] || {
     python3 ../scripts/test/smoketest.py &
     pids+=($!)
+
+    bash ../scripts/test/redup.sh &
+    pids+=($!)
 }
 
 for pid in ${pids[@]}; do

scripts/test/redup.sh

@@ -0,0 +1,104 @@
+#!/bin/bash
+set -e
+
+td=redump-test
+
+st_orig() { cat <<'EOF'
+f1,1,'f1',3660
+f2,1,'f2',3720
+f3,2,'f3',3960
+f4,1,'f4',3840
+f5,1,'f5' -> 'f2',3900
+f6,2,'f6',3960
+f7,1,'f7',4020
+f8,1,'f8',4080
+f9,1,'f9',4140
+EOF
+}
+st_ref() { cat <<'EOF'
+f1,1,'f1',3660
+f2,1,'f2',3720
+f3,1,'f3',3960
+f4,1,'f4',3840
+f5,1,'f5',3900
+f6,1,'f6',3960
+f7,1,'f7',4020
+f8,1,'f8',4080
+f9,1,'f9',4140
+EOF
+}
+st_sym() { cat <<'EOF'
+f1,1,'f1',3660
+f2,1,'f2',3720
+f3,1,'f3',3960
+f4,1,'f4',3840
+f5,1,'f5' -> 'f2',3900
+f6,1,'f6' -> 'f3',3960
+f7,1,'f7' -> 'f4',4020
+f8,1,'f8' -> 'f1',4080
+f9,1,'f9' -> 'f1',4140
+EOF
+}
+st_hard() { cat <<'EOF'
+f1,3,'f1',3600
+f2,2,'f2',3600
+f3,2,'f3',3600
+f4,2,'f4',3600
+f5,2,'f5',3600
+f6,2,'f6',3600
+f7,2,'f7',3600
+f8,3,'f8',3600
+f9,3,'f9',3600
+EOF
+}
+statchk() {
+    (cd $td;stat -c%n,%h,%N,%Y * | sort -n | diff -U9 <($1) -) && return
+    cat $td/* | grep -vE '^f1f2f3f4f2f3f4f1f1$' || return
+    cat $td.l
+    exit 1
+}
+setup() {
+    rm -rf $td
+    mkdir $td
+    th=19700101010 #..M
+    tu=978310800 #unix
+    ( cd "$td"
+        for n in {1..4}; do echo -n f$n > f$n; done
+        ln -s f2 f5
+        ln f3 f6
+        cp --reflink=always f4 f7
+        cp --reflink=never f1 f8
+        cp --reflink=never f1 f9
+        for n in {1..9}; do touch -ht $th$n f$n; done
+    );statchk st_orig
+}
+export PRTY_NO_ARGON2=1
+export PRTY_NO_CFSSL=1
+export PRTY_NO_FFMPEG=1
+export PRTY_NO_FFPROBE=1
+export PRTY_NO_IFADDR=1
+export PRTY_NO_IMPRESO=1
+export PRTY_NO_MAGIC=1
+export PRTY_NO_PARAMIKO=1
+export PRTY_NO_PARTFTPY=1
+export PRTY_NO_PIL=1
+export PRTY_NO_PSUTIL=1
+export PRTY_NO_PYFTPD=1
+export PRTY_NO_RAW=1
+export PRTY_NO_VIPS=1
+
+cmd="python -m copyparty -i no --ign-ebind-all --no-ses --no-fastboot --no-voldump --exit=idx --ansi -v $td::A"
+
+fs=$(stat -fc%T .)
+if [ "$fs" = btrfs ]; then
+    echo ref:;  setup; $cmd -e2dsa --redup sym,hard,ref=ref >$td.l; statchk st_ref
+else
+    printf 'detected filesystem [%s] which is not btrfs, will NOT run reflink test\n' "$fs"
+fi
+
+echo sym:;  setup; $cmd -e2dsa --redup sym,hard,ref=sym >$td.l; statchk st_sym
+
+echo hard:; setup; $cmd -e2dsa --redup sym,hard,ref=hard >$td.l; touch -ht 197001010100 $td/*; statchk st_hard
+# `- ign hard mtimes because redup is order-undefined (or sqlite-row-order rather but ye)
+
+rm -rf $td $td.l

tests/util.py

@@ -145,7 +145,7 @@ class Cfg(Namespace):
     def __init__(self, a=None, v=None, c=None, **ka0):
         ka = {}
 
-        ex = "allow_flac allow_wav allow_svg_js chpw cookie_lax daw dav_auth dav_mac dav_rt dlni dothidden e2d e2ds e2dsa e2t e2ts e2tsr e2v e2vu e2vp early_ban ed emp exp force_js getmod grid gsel hardlink hardlink_only http_no_tcp ih ihead localtime log_badxml magic md_no_br nid nih no_acode no_athumb no_bauth no_clone no_cp no_dav no_db_ip no_del no_dirsz no_dupe no_dupe_m no_fnugg no_html no_lifetime no_logues no_mime no_mv no_pipe no_poll no_readme no_robots no_sb_md no_sb_lg no_scandir no_script no_tail no_tarcmp no_thumb no_vthumb no_u2abrt no_zip no_zls nrand nsort nw og og_no_head og_s_title ohead opds q rand re_dirsz reflink rm_partial rmagic rss show_hist smb srch_dbg srch_excl srch_nfkc srch_icase stats ui_noacci ui_nocpla ui_noctxb ui_nolbar ui_nombar ui_nonav ui_notree ui_norepl ui_nosrvi uqe usernames vague_403 vc ver vol_nospawn vol_or_crash wo_up_readme wopi write_uplog xdev xlink xvol zipmaxu zs"
+        ex = "allow_flac allow_wav allow_svg_js chpw cookie_lax daw dav_auth dav_mac dav_rt dlni dothidden e2d e2ds e2dsa e2t e2ts e2tsr e2v e2vu e2vp early_ban ed emp exp force_js getmod grid gsel hardlink hardlink_only http_no_tcp ih ihead localtime log_badxml magic md_no_br nid nih no_acode no_athumb no_bauth no_clone no_cp no_dav no_db_ip no_del no_dirsz no_dupe no_dupe_m no_fnugg no_html no_lifetime no_logues no_mime no_mv no_pipe no_poll no_readme no_robots no_sb_md no_sb_lg no_scandir no_script no_tail no_tarcmp no_thumb no_vthumb no_u2abrt no_zip no_zls nrand nsort nw og og_no_head og_s_title ohead opds q rand re_dirsz redup_dry reflink rm_partial rmagic rss show_hist smb srch_dbg srch_excl srch_nfkc srch_icase stats ui_noacci ui_nocpla ui_noctxb ui_nolbar ui_nombar ui_nonav ui_notree ui_norepl ui_nosrvi uqe usernames vague_403 vc ver vol_nospawn vol_or_crash wo_up_readme wopi write_uplog xdev xlink xvol zipmaxu zs"
         ka.update(**{k: False for k in ex.split()})
 
         ex = "dav_inf dedup dotpart dotsrch hist_cow hook_v no_dhash no_fastboot no_fpool no_htp no_rescan no_sendfile no_ses no_snap no_up_list no_voldump wram re_dhash see_dots plain_ip"
@@ -166,7 +166,7 @@ class Cfg(Namespace):
         ex = "ctl_re db_act forget_ip gauto idp_cookie idp_store k304 loris md_nhist no304 nosubtle qr_pin qr_wait re_maxage rproxy rsp_jtr rsp_slp s_wr_slp snap_wri theme themes turbo u2ow zipmaxn zipmaxs"
         ka.update(**{k: 0 for k in ex.split()})
 
-        ex = "ah_alg bname chdir chmod_f chpw_db csp_dl csp_ui db_xattr doctitle df epilogues exit favico fika ipa ipar html_head html_head_d html_head_s idp_login idp_logout lg_sba lg_sbf log_date log_fk md_sba md_sbf name og_desc og_site og_th og_title og_title_a og_title_v og_title_i opds_exts preadmes prologues readmes shr shr1 shr_site site smsg tcolor textfiles th_pregen txt_eol ufavico ufavico_h unlist up_site vc_url vname xff_src zipmaxt R RS SR"
+        ex = "ah_alg bname chdir chmod_f chpw_db csp_dl csp_ui db_xattr doctitle df epilogues exit favico fika ipa ipar html_head html_head_d html_head_s idp_login idp_logout lg_sba lg_sbf log_date log_fk md_sba md_sbf name og_desc og_site og_th og_title og_title_a og_title_v og_title_i opds_exts preadmes prologues readmes redup shr shr1 shr_site site smsg tcolor textfiles th_pregen txt_eol ufavico ufavico_h unlist up_site vc_url vname xff_src zipmaxt R RS SR"
         ka.update(**{k: "" for k in ex.split()})
 
         ex = "apnd_who ban_403 ban_404 ban_422 ban_pw ban_pwc ban_url dont_ban cachectl http_vary rcm rss_fmt_d rss_fmt_t spinner"