From fd4f434192f4e6388a47aa3749db816d366db9da Mon Sep 17 00:00:00 2001 From: ed Date: Wed, 16 Sep 2026 22:33:38 +0000 Subject: [PATCH] add --redup --- copyparty/__main__.py | 2 + copyparty/authsrv.py | 12 +++ copyparty/cfg.py | 4 + copyparty/up2k.py | 168 +++++++++++++++++++++++++++++++++++++----- copyparty/util.py | 1 + scripts/run-tests.sh | 3 + scripts/test/redup.sh | 104 ++++++++++++++++++++++++++ tests/util.py | 4 +- 8 files changed, 278 insertions(+), 20 deletions(-) create mode 100755 scripts/test/redup.sh diff --git a/copyparty/__main__.py b/copyparty/__main__.py index 5571dd669..9a19e61ef 100644 --- a/copyparty/__main__.py +++ b/copyparty/__main__.py @@ -1437,6 +1437,8 @@ def add_upload(ap): ap2.add_argument("--hardlink", action="store_true", help="enable hardlink-based dedup; will fallback on symlinks when that is impossible (across filesystems) (volflag=hardlink)") ap2.add_argument("--hardlink-only", action="store_true", help="do not fallback to symlinks when a hardlink cannot be made (volflag=hardlinkonly)") ap2.add_argument("--reflink", action="store_true", help="enable reflink-based dedup; will fallback on full copies when that is impossible (non-CoW filesystem) (volflag=reflink)") + ap2.add_argument("--redup", metavar="A[,B]=T", help="convert dedup-type for existing files; \033[33mA,B\033[0m is dedup-types to convert from (no/ref/sym/hard), \033[33mT\033[0m is target type; converting from sym/hard is fast-ish, from no/ref is slooow; example: [\033[32msym,hard=ref\033[0m] (volflag=redup)") + ap2.add_argument("--redup-dry", action="store_true", help="dry-run; makes \033[33m--redup\033[0m not apply changes (volflag=redup_dry)") ap2.add_argument("--no-dupe", action="store_true", help="reject duplicate files during upload; only matches within the same volume (volflag=nodupe)") ap2.add_argument("--no-dupe-m", action="store_true", help="also reject dupes when moving a file into another volume (volflag=nodupem)") ap2.add_argument("--no-clone", action="store_true", help="do not use existing data on disk to satisfy dupe uploads; reduces server HDD reads in exchange for much more network load (volflag=noclone)") diff --git a/copyparty/authsrv.py b/copyparty/authsrv.py index c522e1617..e74dc7955 100644 --- a/copyparty/authsrv.py +++ b/copyparty/authsrv.py @@ -107,6 +107,7 @@ SEESLOG = " (see fileserver log for details)" SSEELOG = " ({})".format(SEE_LOG) BAD_CFG = "invalid config; {}".format(SEE_LOG) SBADCFG = " ({})".format(BAD_CFG) +REDUP_E2 = "WARNING: [/%s] needs 'e2ds' for redup from 'ref' and/or to 'sym'" PTN_U_GRP = re.compile(r"\$\{u(%[+-][^}]+)\}") PTN_G_GRP = re.compile(r"\$\{g(%[+-][^}]+)\}") @@ -2915,6 +2916,17 @@ class AuthSrv(object): if ccs == "y": vol.flags["bcasechk"] = True + ptn = re.compile(r"^(,no|,sym|,hard|,ref)+=(sym|hard|ref)$") + ptn2 = re.compile(r"(no|ref).*=|=sym") + for vol in vfs.all_nodes.values(): + zs = vol.flags.get("redup") + if zs and not ptn.match("," + zs): + t = "invalid redup for volume /%s: %r" + self.log(t % (vol.vpath, zs), 1) + errors = True + if zs and "e2ds" not in vol.flags and ptn2.search(zs): + self.log(REDUP_E2 % (vol.vpath,), 3) + tags = self.args.mtp or [] tags = [x.split("=")[0] for x in tags] tags = [y for x in tags for y in x.split(",")] diff --git a/copyparty/cfg.py b/copyparty/cfg.py index d70b0bd40..09e965ef4 100644 --- a/copyparty/cfg.py +++ b/copyparty/cfg.py @@ -66,6 +66,7 @@ def vf_bmap() -> dict[str, str]: "og_s_title", "opds", "rand", + "redup_dry", "reflink", "rm_partial", "rmagic", @@ -146,6 +147,7 @@ def vf_vmap() -> dict[str, str]: "put_ck", "put_name", "readmes", + "redup", "rotf_tz", "mv_retry", "rm_retry", @@ -229,6 +231,8 @@ flagcats = { "hardlinkonly": "dedup with hardlink only, never symlink;\nmake a full copy if hardlink is impossible", "reflink": "enable reflink-based file deduplication,\nwith fallback on full copy when that is impossible", "safededup": "verify on-disk data before using it for dedup", + "redup=A,B=T": "convert dedup-type for existing files", + "redup_dry": "dry-run mode for redup", "noclone": "take dupe data from clients, even if available on HDD", "nodupe": "rejects existing files (instead of linking/cloning them)", "nodupem": "rejects existing files during moves as well", diff --git a/copyparty/up2k.py b/copyparty/up2k.py index 8bb62d0df..da65f2086 100644 --- a/copyparty/up2k.py +++ b/copyparty/up2k.py @@ -17,7 +17,7 @@ from copy import deepcopy from queue import Queue from .__init__ import ANYWIN, PY2, TYPE_CHECKING, UNIX, WINDOWS, E -from .authsrv import LEELOO_DALLAS, SEESLOG, VFS, AuthSrv +from .authsrv import LEELOO_DALLAS, REDUP_E2, SEESLOG, VFS, AuthSrv from .bos import bos from .cfg import vf_bmap, vf_cmap, vf_vmap from .fsutil import Fstab @@ -1000,6 +1000,21 @@ class Up2k(object): with self.mutex, self.reg_mutex: self._drop_caches() + def setstate(vol, n): + t = "" + if n < 1 and "e2v" in vol.flags: + t += "+e2v" + if n < 2 and "redup" in vol.flags: + t += "+redup" + if n < 3 and "e2ts" in vol.flags: + t += "+tags" + + if t: + t = "online (%s pending)" % (t[1:],) + else: + t = "online, idle" + self.volstate[vol.vpath] = t + for vol in vols: if self.stop or gid != self.gid: break @@ -1016,26 +1031,16 @@ class Up2k(object): if vac: need_vac[vol] = True - if "e2v" in vol.flags: - t = "online (integrity-check pending)" - elif "e2ts" in vol.flags: - t = "online (tags pending)" - else: - t = "online, idle" - - self.volstate[vol.vpath] = t + setstate(vol, 0) self._unblock() # file contents verification for vol in vols: - if self.stop: - break - - if "e2v" not in vol.flags: + if self.stop or "e2v" not in vol.flags: continue - t = "online (verifying integrity)" + t = "online (doing e2v)" self.volstate[vol.vpath] = t self.log("{} [{}]".format(t, vol.realpath)) @@ -1044,12 +1049,23 @@ class Up2k(object): self.log("modified {} entries in the db".format(nmod), 3) need_vac[vol] = True - if "e2ts" in vol.flags: - t = "online (tags pending)" - else: - t = "online, idle" + setstate(vol, 1) + # convert dedup type + for vol in vols: + if self.stop or not vol.flags.get("redup"): + continue + + t = "online (doing redup)" self.volstate[vol.vpath] = t + self.log("{} [{}]".format(t, vol.realpath)) + + try: + with self.mutex, self.reg_mutex: + self._redup(vol, vols) + except: + self.log("redup failed: " + min_ex(), 1) + setstate(vol, 2) # open the rest + do any e2ts(a) needed_mutagen = False @@ -2088,6 +2104,122 @@ class Up2k(object): return len(rewark) + len(f404) + def _redup(self, vol: VFS, vols: list[VFS]) -> None: + logmsg = "redup: replacing %r with %r" + dry = "redup_dry" in vol.flags + if dry: + logmsg += " #dry" + + zs, to = vol.flags["redup"].split("=") + zsl = zs.split(",") + cref = "no" in zsl or "ref" in zsl + csym = "sym" in zsl + chard = "hard" in zsl + chard0 = chard + + vf = vol.flags.copy() + vf["dedup"] = 1 + for zs in "hardlink hardlinkonly reflink".split(): + vf.pop(zs, None) + + if to == "ref": + vf["reflink"] = 1 + elif to == "hard": + vf["hardlinkonly"] = vf["hardlink"] = 1 + elif to == "sym": + pass + else: + raise Exception("target type not implemented: " + to) + + if (csym or chard) and to != "sym": + self.log("redup: stage 1 begin") + if to == "hard": + chard = False + for x in vol.walk("", "", [], LEELOO_DALLAS, [[]], 2, True, True, True): + vn, _, _, atop, lsf, lsd, lsv = x + if vn is not vol: + lsd[:] = [] + lsv.clear() + continue + for fn, st in lsf: + if (csym and stat.S_ISLNK(st.st_mode)) or ( + chard and st.st_nlink > 1 + ): + ap = os.path.join(atop, fn) + self.log(logmsg % (ap, absreal(ap))) + if dry: + continue + ap2 = tempfile.NamedTemporaryFile( + prefix="r,", dir=atop, delete=False + ).name + self._symlink(ap, ap2, vf, True, True, st.st_mtime) + wunlink(self.log, ap, vf) + bos.rename(ap2, ap) + + if cref or to == "sym": + self.log("redup: stage 2 begin") + if "e2ds" not in vol.flags: + self.log(REDUP_E2 % (vol.vpath,), 3) + cur = self.cur[vol.realpath] + curs = [(vol, cur.connection.cursor())] + if to != "sym": + for v2 in vols: + if vol is not v2 and v2.realpath in self.cur: + curs += [(v2, self.cur[v2.realpath])] + nrem = cur.execute("select count(w) from up").fetchone()[0] + self.log("redup [/%s]: %d files left" % (vol.vpath, nrem)) + q2 = "select rd, fn from up where substr(w,1,16) = ? and w=?" + for w, rd, fn in cur.execute("select w, rd, fn from up"): + w16 = w[:16] + for v2, c2 in curs: + hit = c2.execute(q2, (w16, w)).fetchone() + if not hit: + continue + rd2, fn2 = hit + if fn == fn2 and rd == rd2 and vol is v2: + continue + try: + apt = "" + rd, fn = s3dec(rd, fn) + rd2, fn2 = s3dec(rd2, fn2) + fp1 = os.path.join(vol.realpath, rd, fn) + fp2 = os.path.join(v2.realpath, rd2, fn2) + st1 = bos.lstat(fp1) + if fp1 == fp2 or stat.S_ISLNK(st1.st_mode): + continue # self(?), or redup failed during walk + ap1 = absreal(fp1) + ap2 = absreal(fp2) + st2 = bos.lstat(ap2) + if stat.S_ISLNK(st2.st_mode): + continue # dead hit + for ap in (ap1,) if ap1 == ap2 else (ap1, ap2): + self.log("redup: integrity-checking %r" % (ap,)) + zsl, st = self._hashlist_from_file(ap) + w2 = up2k_wark_from_hashlist(self.salt, st.st_size, zsl) + if w2 != w: + t = "db desync:\n%s %r\n%s db" + raise Exception(t % (w2, ap, w)) + + self.log(logmsg % (ap1, ap2)) + if dry: + continue + apt = tempfile.NamedTemporaryFile( + prefix="r,", dir=os.path.dirname(fp1), delete=False + ).name + self._symlink(ap2, apt, vf, True, True, st1.st_mtime) + zsl, st = self._hashlist_from_file(apt) + w2 = up2k_wark_from_hashlist(self.salt, st.st_size, zsl) + if w != w2: + raise Exception("bad checksum after copy?!") + wunlink(self.log, ap1, vf) + bos.rename(apt, ap1) + break + except Exception as ex: + self.log("redup: skipping match due to " + str(ex)) + if apt and os.path.exists(apt): + os.unlink(apt) + continue + def _build_tags_index(self, vol: VFS) -> tuple[int, int, bool]: ptop = vol.realpath with self.mutex, self.reg_mutex: diff --git a/copyparty/util.py b/copyparty/util.py index a38fe1705..62cffdaa1 100644 --- a/copyparty/util.py +++ b/copyparty/util.py @@ -450,6 +450,7 @@ IMPLICATIONS = [ ["no_logue", "no_logues"], # user-typo ["nologues", "no_logues"], # user-typo ["nologue", "no_logues"], # user-typo + ["nw", "redup_dry"], ["plainlogue", "plainlogues"], # user-typo ["plainreadmes", "plainreadme"], # user-typo ["noscript", "plainlogues"], diff --git a/scripts/run-tests.sh b/scripts/run-tests.sh index 644ca6bfa..ecb77b98a 100755 --- a/scripts/run-tests.sh +++ b/scripts/run-tests.sh @@ -70,6 +70,9 @@ done [ "$1" ] || { python3 ../scripts/test/smoketest.py & pids+=($!) + + bash ../scripts/test/redup.sh & + pids+=($!) } for pid in ${pids[@]}; do diff --git a/scripts/test/redup.sh b/scripts/test/redup.sh new file mode 100755 index 000000000..6310ce1ac --- /dev/null +++ b/scripts/test/redup.sh @@ -0,0 +1,104 @@ +#!/bin/bash +set -e + +td=redump-test + +st_orig() { cat <<'EOF' +f1,1,'f1',3660 +f2,1,'f2',3720 +f3,2,'f3',3960 +f4,1,'f4',3840 +f5,1,'f5' -> 'f2',3900 +f6,2,'f6',3960 +f7,1,'f7',4020 +f8,1,'f8',4080 +f9,1,'f9',4140 +EOF +} +st_ref() { cat <<'EOF' +f1,1,'f1',3660 +f2,1,'f2',3720 +f3,1,'f3',3960 +f4,1,'f4',3840 +f5,1,'f5',3900 +f6,1,'f6',3960 +f7,1,'f7',4020 +f8,1,'f8',4080 +f9,1,'f9',4140 +EOF +} +st_sym() { cat <<'EOF' +f1,1,'f1',3660 +f2,1,'f2',3720 +f3,1,'f3',3960 +f4,1,'f4',3840 +f5,1,'f5' -> 'f2',3900 +f6,1,'f6' -> 'f3',3960 +f7,1,'f7' -> 'f4',4020 +f8,1,'f8' -> 'f1',4080 +f9,1,'f9' -> 'f1',4140 +EOF +} +st_hard() { cat <<'EOF' +f1,3,'f1',3600 +f2,2,'f2',3600 +f3,2,'f3',3600 +f4,2,'f4',3600 +f5,2,'f5',3600 +f6,2,'f6',3600 +f7,2,'f7',3600 +f8,3,'f8',3600 +f9,3,'f9',3600 +EOF +} +statchk() { + (cd $td;stat -c%n,%h,%N,%Y * | sort -n | diff -U9 <($1) -) && return + cat $td/* | grep -vE '^f1f2f3f4f2f3f4f1f1$' || return + cat $td.l + exit 1 +} +setup() { + rm -rf $td + mkdir $td + th=19700101010 #..M + tu=978310800 #unix + ( cd "$td" + for n in {1..4}; do echo -n f$n > f$n; done + ln -s f2 f5 + ln f3 f6 + cp --reflink=always f4 f7 + cp --reflink=never f1 f8 + cp --reflink=never f1 f9 + for n in {1..9}; do touch -ht $th$n f$n; done + );statchk st_orig +} +export PRTY_NO_ARGON2=1 +export PRTY_NO_CFSSL=1 +export PRTY_NO_FFMPEG=1 +export PRTY_NO_FFPROBE=1 +export PRTY_NO_IFADDR=1 +export PRTY_NO_IMPRESO=1 +export PRTY_NO_MAGIC=1 +export PRTY_NO_PARAMIKO=1 +export PRTY_NO_PARTFTPY=1 +export PRTY_NO_PIL=1 +export PRTY_NO_PSUTIL=1 +export PRTY_NO_PYFTPD=1 +export PRTY_NO_RAW=1 +export PRTY_NO_VIPS=1 + +cmd="python -m copyparty -i no --ign-ebind-all --no-ses --no-fastboot --no-voldump --exit=idx --ansi -v $td::A" + +fs=$(stat -fc%T .) +if [ "$fs" = btrfs ]; then + echo ref:; setup; $cmd -e2dsa --redup sym,hard,ref=ref >$td.l; statchk st_ref +else + printf 'detected filesystem [%s] which is not btrfs, will NOT run reflink test\n' "$fs" +fi + +echo sym:; setup; $cmd -e2dsa --redup sym,hard,ref=sym >$td.l; statchk st_sym + +echo hard:; setup; $cmd -e2dsa --redup sym,hard,ref=hard >$td.l; touch -ht 197001010100 $td/*; statchk st_hard +# `- ign hard mtimes because redup is order-undefined (or sqlite-row-order rather but ye) + +rm -rf $td $td.l diff --git a/tests/util.py b/tests/util.py index f69448557..abb60dfce 100644 --- a/tests/util.py +++ b/tests/util.py @@ -145,7 +145,7 @@ class Cfg(Namespace): def __init__(self, a=None, v=None, c=None, **ka0): ka = {} - ex = "allow_flac allow_wav allow_svg_js chpw cookie_lax daw dav_auth dav_mac dav_rt dlni dothidden e2d e2ds e2dsa e2t e2ts e2tsr e2v e2vu e2vp early_ban ed emp exp force_js getmod grid gsel hardlink hardlink_only http_no_tcp ih ihead localtime log_badxml magic md_no_br nid nih no_acode no_athumb no_bauth no_clone no_cp no_dav no_db_ip no_del no_dirsz no_dupe no_dupe_m no_fnugg no_html no_lifetime no_logues no_mime no_mv no_pipe no_poll no_readme no_robots no_sb_md no_sb_lg no_scandir no_script no_tail no_tarcmp no_thumb no_vthumb no_u2abrt no_zip no_zls nrand nsort nw og og_no_head og_s_title ohead opds q rand re_dirsz reflink rm_partial rmagic rss show_hist smb srch_dbg srch_excl srch_nfkc srch_icase stats ui_noacci ui_nocpla ui_noctxb ui_nolbar ui_nombar ui_nonav ui_notree ui_norepl ui_nosrvi uqe usernames vague_403 vc ver vol_nospawn vol_or_crash wo_up_readme wopi write_uplog xdev xlink xvol zipmaxu zs" + ex = "allow_flac allow_wav allow_svg_js chpw cookie_lax daw dav_auth dav_mac dav_rt dlni dothidden e2d e2ds e2dsa e2t e2ts e2tsr e2v e2vu e2vp early_ban ed emp exp force_js getmod grid gsel hardlink hardlink_only http_no_tcp ih ihead localtime log_badxml magic md_no_br nid nih no_acode no_athumb no_bauth no_clone no_cp no_dav no_db_ip no_del no_dirsz no_dupe no_dupe_m no_fnugg no_html no_lifetime no_logues no_mime no_mv no_pipe no_poll no_readme no_robots no_sb_md no_sb_lg no_scandir no_script no_tail no_tarcmp no_thumb no_vthumb no_u2abrt no_zip no_zls nrand nsort nw og og_no_head og_s_title ohead opds q rand re_dirsz redup_dry reflink rm_partial rmagic rss show_hist smb srch_dbg srch_excl srch_nfkc srch_icase stats ui_noacci ui_nocpla ui_noctxb ui_nolbar ui_nombar ui_nonav ui_notree ui_norepl ui_nosrvi uqe usernames vague_403 vc ver vol_nospawn vol_or_crash wo_up_readme wopi write_uplog xdev xlink xvol zipmaxu zs" ka.update(**{k: False for k in ex.split()}) ex = "dav_inf dedup dotpart dotsrch hist_cow hook_v no_dhash no_fastboot no_fpool no_htp no_rescan no_sendfile no_ses no_snap no_up_list no_voldump wram re_dhash see_dots plain_ip" @@ -166,7 +166,7 @@ class Cfg(Namespace): ex = "ctl_re db_act forget_ip gauto idp_cookie idp_store k304 loris md_nhist no304 nosubtle qr_pin qr_wait re_maxage rproxy rsp_jtr rsp_slp s_wr_slp snap_wri theme themes turbo u2ow zipmaxn zipmaxs" ka.update(**{k: 0 for k in ex.split()}) - ex = "ah_alg bname chdir chmod_f chpw_db csp_dl csp_ui db_xattr doctitle df epilogues exit favico fika ipa ipar html_head html_head_d html_head_s idp_login idp_logout lg_sba lg_sbf log_date log_fk md_sba md_sbf name og_desc og_site og_th og_title og_title_a og_title_v og_title_i opds_exts preadmes prologues readmes shr shr1 shr_site site smsg tcolor textfiles th_pregen txt_eol ufavico ufavico_h unlist up_site vc_url vname xff_src zipmaxt R RS SR" + ex = "ah_alg bname chdir chmod_f chpw_db csp_dl csp_ui db_xattr doctitle df epilogues exit favico fika ipa ipar html_head html_head_d html_head_s idp_login idp_logout lg_sba lg_sbf log_date log_fk md_sba md_sbf name og_desc og_site og_th og_title og_title_a og_title_v og_title_i opds_exts preadmes prologues readmes redup shr shr1 shr_site site smsg tcolor textfiles th_pregen txt_eol ufavico ufavico_h unlist up_site vc_url vname xff_src zipmaxt R RS SR" ka.update(**{k: "" for k in ex.split()}) ex = "apnd_who ban_403 ban_404 ban_422 ban_pw ban_pwc ban_url dont_ban cachectl http_vary rcm rss_fmt_d rss_fmt_t spinner"