diff --git a/copyparty/__main__.py b/copyparty/__main__.py index f3d625c59..cb367988a 100644 --- a/copyparty/__main__.py +++ b/copyparty/__main__.py @@ -850,7 +850,7 @@ def get_sects(): ], [ "dedup", - "file deduplication", + "file deduplication on upload", dedent( """ when just \033[36m--dedup\033[0m or volflag "\033[36mdedup\033[0m" is enabled, files will be @@ -860,11 +860,13 @@ def get_sects(): * symlinks work on most filesystems, but if another software moves either the symlink itself or the file that it points to, - the link will break and the file "disappears" + the link will break and the file "disappears"; + only safe if only copyparty is used to rename/move/delete * hardlinks work on Linux filesystems and \033[1mlook\033[0m like regular files, except if the contents of one copy of that file is edited, then this will affect \033[1mall the dupes too!\033[0m + and they share the same last-modified-time * reflinks only work on btrfs/zfs/xfs but is the safest/best choice; deduplicated files behave just like regular files, @@ -882,6 +884,33 @@ def get_sects(): if hardlink is not possible (unsupp/cross-fs), a symlink is made note: dedup works best when combined with e2ds or e2dsa + + to deduplicate existing files on-disk, see \033[36m--help-redup\033[0m + """ + ), + ], + [ + "redup", + "deduplicating existing files on disk", + dedent( + """ + \033[36m--redup\033[0m can convert on-disk files between deduplication-approaches, + for example turning redundant full copies into reflinks to save space + + example: --redup \033[32mno,sym,hard\033[0m=\033[35mref\033[0m will find all files of type + \033[32mfullcopy/symlink/hardlink\033[0m, and convert them into \033[35mreflinks\033[0m + (see \033[36m--help-dedup\033[0m for a summary on the types) + + converting \033[1mfrom\033[0m symlinks/hardlinks is fairly fast and simple; + in this case, the obvious target type/approach is reflinks + which only works on btrfs/zfs/xfs + + converting \033[1mto\033[0m symlinks or hardlinks is \033[3mpossible\033[0m, but + not-recommended due to risks explained in \033[36m--help-dedup\033[0m + (interaction with other softwares can result in data loss) + + converting \033[1mfrom\033[0m 'no' (full redundant copies) is hella slow; + checksums each file to avoid corruption from bad RAM """ ), ], @@ -1475,7 +1504,7 @@ def add_upload(ap): ap2.add_argument("--hardlink", action="store_true", help="enable hardlink-based dedup; will fallback on symlinks when that is impossible (across filesystems) (volflag=hardlink)") ap2.add_argument("--hardlink-only", action="store_true", help="do not fallback to symlinks when a hardlink cannot be made (volflag=hardlinkonly)") ap2.add_argument("--reflink", action="store_true", help="enable reflink-based dedup; will fallback on full copies when that is impossible (non-CoW filesystem) (volflag=reflink)") - ap2.add_argument("--redup", metavar="A[,B]=T", help="convert dedup-type for existing files; \033[33mA,B\033[0m is dedup-types to convert from (no/ref/sym/hard), \033[33mT\033[0m is target type; converting from sym/hard is fast-ish, from no/ref is slooow; see \033[33m--help-dedup\033[0m. Example: [\033[32msym,hard=ref\033[0m] (volflag=redup)") + ap2.add_argument("--redup", metavar="A[,B]=T", help="convert dedup-type for existing files; \033[33mA,B\033[0m is dedup-types to convert from (no/ref/sym/hard), \033[33mT\033[0m is target type; converting from sym/hard is fast-ish, from no/ref is slooow, to sym/hard is jank; see \033[33m--help-redup\033[0m. Example: [\033[32msym,hard=ref\033[0m] (volflag=redup)") ap2.add_argument("--redup-dry", action="store_true", help="dry-run; makes \033[33m--redup\033[0m not apply changes (volflag=redup_dry)") ap2.add_argument("--no-dupe", action="store_true", help="reject duplicate files during upload; only matches within the same volume (volflag=nodupe)") ap2.add_argument("--no-dupe-m", action="store_true", help="also reject dupes when moving a file into another volume (volflag=nodupem)") diff --git a/copyparty/up2k.py b/copyparty/up2k.py index 85ca27c57..d33496073 100644 --- a/copyparty/up2k.py +++ b/copyparty/up2k.py @@ -2157,6 +2157,11 @@ class Up2k(object): prefix="r,", dir=atop, delete=False ).name self._symlink(ap, ap2, vf, True, True, st.st_mtime) + st1 = bos.stat(ap) + st2 = bos.lstat(ap2) + if st1.st_size != st2.st_size or stat.S_ISLNK(st2.st_mode): + wunlink(self.log, ap2, vf) + raise Exception("redup: clone failed; giving up") wunlink(self.log, ap, vf) bos.rename(ap2, ap) diff --git a/scripts/genhelp.sh b/scripts/genhelp.sh index 4ec08c0f9..ea6be03a1 100755 --- a/scripts/genhelp.sh +++ b/scripts/genhelp.sh @@ -15,7 +15,7 @@ command -v gsort >/dev/null && { cd ../dist -kwds='-bind -accounts -auth -auth-ord -flags -dedup -thumb-ex -handlers -hooks -idp -urlform -exp -rlo -ls -dbd -chmod -pwhash -zm' +kwds='-bind -accounts -auth -auth-ord -flags -dedup -redup -thumb-ex -handlers -hooks -idp -urlform -exp -rlo -ls -dbd -chmod -pwhash -zm' export PRTY_FULL_HELP=1 export PRTY_NO_PARAMIKO=1 diff --git a/scripts/help2html.py b/scripts/help2html.py index 793afb31a..14deb2881 100755 --- a/scripts/help2html.py +++ b/scripts/help2html.py @@ -8,7 +8,7 @@ import subprocess as sp # to convert the copyparty --help to html, run this in xfce4-terminal @ 140x43: _ = r"""" -echo; for a in '' -bind -accounts -auth -auth-ord -flags -dedup -thumb-ex -handlers -hooks -idp -urlform -exp -rlo -ls -dbd -chmod -pwhash -zm; do +echo; for a in '' -bind -accounts -auth -auth-ord -flags -dedup -redup -thumb-ex -handlers -hooks -idp -urlform -exp -rlo -ls -dbd -chmod -pwhash -zm; do ./copyparty-sfx.py --help$a 2>/dev/null; printf '\n\n\n%0139d\n\n\n'; done # xfce4-terminal @ 140x43 """ # click [edit] => [select all] diff --git a/scripts/help2txt.sh b/scripts/help2txt.sh index 83e31f52d..ba198185c 100755 --- a/scripts/help2txt.sh +++ b/scripts/help2txt.sh @@ -29,7 +29,7 @@ exit 0 # first open an infinitely wide console (this is why you own an ultrawide) and copypaste this into it: -for a in '' -bind -accounts -auth -auth-ord -flags -dedup -thumb-ex -handlers -hooks -idp -urlform -exp -rlo -ls -dbd -chmod -pwhash -zm; do +for a in '' -bind -accounts -auth -auth-ord -flags -dedup -redup -thumb-ex -handlers -hooks -idp -urlform -exp -rlo -ls -dbd -chmod -pwhash -zm; do ./copyparty-sfx.py --help$a 2>/dev/null; printf '\n\n\n%0255d\n\n\n'; done # then copypaste all of the output by pressing ctrl-shift-a, ctrl-shift-c