fix(nix): separate makeWrapperArgs properly (#1676)

* fix(nix): separate makeWrapperArgs properly

Since Python packages are now built with `__structuredAttrs = true;` in nixpkgs, this is neccesary.

See also: https://github.com/NixOS/nixpkgs/commit/a8383b633210925071f3d7c680c8d901939c7fd5
See also: https://github.com/NixOS/nixpkgs/pull/554025

* chore(nix): nixfmt

.gitattributes

@@ -4,3 +4,5 @@
 
 *.png binary
 *.gif binary
+*.ico binary
+*.gz binary

.github/ISSUE_TEMPLATE/bug_report.md

@@ -7,10 +7,11 @@ assignees: '9001'
 
 ---
 
-NOTE:
+<!-- NOTE:
 **please use english, or include an english translation.** aside from that,  
-all of the below are optional, consider them as inspiration, delete and rewrite at will, thx md
-
+include all of the information that might possibly be necessary to help diagnose your issue,
+especially things like the server config and server logs; that said,
+any parts below which are obviously and definitely irrelevant can be skipped -->
 
 ### Describe the bug
 a description of what the bug is
@@ -45,5 +46,11 @@ if the issue is possibly on the client-side, then mention some of the following:
 * OS version: 
 * browser version: 
 
+### The rest of the stack
+if you are connecting directly to copyparty then that's cool, otherwise please mention everything else between copyparty and the browser (reverseproxy, tunnels, etc.)
+
+### Server log
+if the issue might be server-related, include everything that appears in the copyparty log during startup, and also anything else you think might be relevant
+
 ### Additional context
 any other context about the problem here

.github/ISSUE_TEMPLATE/feature_request.md

@@ -7,9 +7,9 @@ assignees: '9001'
 
 ---
 
-NOTE:
+<!-- NOTE:
 **please use english, or include an english translation.** aside from that,  
-all of the below are optional, consider them as inspiration, delete and rewrite at will
+all of the below are optional, consider them as inspiration, delete and rewrite at will -->
 
 **is your feature request related to a problem? Please describe.**
 a description of what the problem is, for example, `I'm always frustrated when [...]` or `Why is it not possible to [...]`

.gitignore

@@ -29,8 +29,13 @@ copyparty.egg-info/
 copyparty/res/COPYING.txt
 copyparty/web/deps/
 srv/
+scripts/docker/base/b/
+scripts/docker/base/cver*
+scripts/docker/base/test-aac/
+scripts/docker/base/whl/
 scripts/docker/i/
 scripts/deps-docker/uncomment.py
+scripts/deps-docker/unhint.py
 contrib/package/arch/pkg/
 contrib/package/arch/src/
 
@@ -43,6 +48,7 @@ scripts/docker/*.err
 
 # nix build output link
 result
+result-*
 
 # IDEA config
 .idea/

.vscode/launch.json

@@ -3,7 +3,7 @@
     "configurations": [
         {
             "name": "Run copyparty",
-            "type": "python",
+            "type": "debugpy",
             "request": "launch",
             "module": "copyparty",
             "console": "integratedTerminal",
@@ -11,30 +11,46 @@
             "justMyCode": false,
             "env": {
                 "PYDEVD_DISABLE_FILE_VALIDATION": "1",
-                "PYTHONWARNINGS": "always", //error
+                "PYTHONWARNINGS": "always"  //error
             },
             "args": [
-                //"-nw",
-                "-ed",
-                "-emp",
+                //"-nw",  // no-write; for testing uploads without writing to disk
+                //"-q",  // quiet; speedboost when console output is not needed
+
+                // # increase debugger performance:
+                //"no-htp",
+                //"hash-mt=0",
+                //"mtag-mt=1",
+                //"th-mt=1",
+
+                // # listen for FTP and TFTP
+                "--ftp=3921",
+                "--ftp-pr=12000-12099",
+                "--tftp=3969",
+
+                // # listen on all IPv6, all IPv4, and unix-socket
+                "-i::,unix:777:a.sock",
+
+                // # misc
+                "--dedup",
                 "-e2dsa",
                 "-e2ts",
-                "-mtp=.bpm=f,bin/mtag/audio-bpm.py",
+                "--rss",
+                "--shr=/shr",
+                "--stats",
+                "-z",
+
+                // # users + volumes
                 "-aed:wark",
-                "-vsrv::r:rw,ed:c,dupe",
-                "-vdist:dist:r"
+                "-vdist:dist:r",
+                "-vsrv::r:rw,ed",
+                "-vsrv/junk:junk:r:A,ed",
+                "--ver"
             ]
         },
-        {
-            "name": "No debug",
-            "preLaunchTask": "no_dbg",
-            "type": "python",
-            //"request": "attach", "port": 42069
-            // fork: nc -l 42069 </dev/null
-        },
         {
             "name": "Run active unit test",
-            "type": "python",
+            "type": "debugpy",
             "request": "launch",
             "module": "unittest",
             "console": "integratedTerminal",
@@ -51,6 +67,6 @@
             "program": "${file}",
             "console": "integratedTerminal",
             "justMyCode": false
-        },
+        }
     ]
-}
+}

CONTRIBUTING.md

@@ -6,13 +6,13 @@ but please:
 
 
 
-# do not use AI / LMM when writing code
+# do not use AI / LLM when writing code
 
 copyparty is 100% organic, free-range, human-written software!
 
 > ⚠ you are now entering a no-copilot zone
 
-the *only* place where LMM/AI *may* be accepted is for [localization](https://github.com/9001/copyparty/tree/hovudstraum/docs/rice#translations) if you are fluent and have confirmed that the translation is accurate.
+the *only* place where LLM/AI *may* be accepted is for [localization](https://github.com/9001/copyparty/tree/hovudstraum/docs/rice#translations) if you are fluent and have confirmed that the translation is accurate.
 
 sorry for the harsh tone, but this is important to me 🙏
 
@@ -39,7 +39,7 @@ if you wanna have a go at coding it up yourself then maybe mention the idea on d
 
 aside from documentation and ideas, some other things that would be cool to have some help with is:
 
-* **translations** -- the copyparty web-UI has translations for english and norwegian at the top of [browser.js](https://github.com/9001/copyparty/blob/hovudstraum/copyparty/web/browser.js); if you'd like to add a translation for another language then that'd be welcome! and if that language has a grammar that doesn't fit into the way the strings are assembled, then we'll fix that as we go :>
+* **translations** -- the copyparty web-UI has translations in [copyparty/web/tl](https://github.com/9001/copyparty/tree/hovudstraum/copyparty/web/tl); if you'd like to [add a translation](https://github.com/9001/copyparty/tree/hovudstraum/docs/rice#translations) for another language then that'd be welcome! and if that language has a grammar that doesn't fit into the way the strings are assembled, then we'll fix that as we go :>
 
   * but please note that support for [RTL (Right-to-Left) languages](https://en.wikipedia.org/wiki/Right-to-left_script) is currently not planned, since the javascript is a bit too jank for that
 

SECURITY.md

@@ -1,9 +1,7 @@
 # Security Policy
 
-if you hit something extra juicy pls let me know on either of the following
+if you hit something extra juicy pls let me know on one of the following:
 * email -- `copyparty@ocv.ze` except `ze` should be `me`
-* [mastodon dm](https://layer8.space/@tripflag) -- `@tripflag@layer8.space`
 * [github private vulnerability report](https://github.com/9001/copyparty/security/advisories/new), wow that form is complicated
-* [twitter dm](https://twitter.com/tripflag) (if im somehow not banned yet)
 
 no bug bounties sorry! all i can offer is greetz in the release notes

bin/README.md

@@ -1,7 +1,7 @@
 # [`u2c.py`](u2c.py)
 * command-line up2k client [(webm)](https://ocv.me/stuff/u2cli.webm)
 * file uploads, file-search, autoresume of aborted/broken uploads
-* sync local folder to server
+* [sync local folder to server](https://github.com/9001/copyparty/#folder-sync)
 * generally faster than browsers
 * if something breaks just restart it
 
@@ -26,12 +26,12 @@ and consider using [../docs/rclone.md](../docs/rclone.md) instead; usually a bit
 ## to run this on windows:
 * install [winfsp](https://github.com/billziss-gh/winfsp/releases/latest) and [python 3](https://www.python.org/downloads/)
   * [x] add python 3.x to PATH (it asks during install)
-* `python -m pip install --user fusepy` (or grab a copy of `fuse.py` from the `connect` page on your copyparty, and keep it in the same folder)
+* `python -m pip install --user mfusepy` (or grab a copy of `mfusepy.py` from the `connect` page on your copyparty, and keep it in the same folder)
 * `python ./partyfuse.py n: http://192.168.1.69:3923/`
 
 10% faster in [msys2](https://www.msys2.org/), 700% faster if debug prints are enabled:
 * `pacman -S mingw64/mingw-w64-x86_64-python{,-pip}`
-* `/mingw64/bin/python3 -m pip install --user fusepy`
+* `/mingw64/bin/python3 -m pip install --user mfusepy`
 * `/mingw64/bin/python3 ./partyfuse.py [...]`
 
 you could replace winfsp with [dokan](https://github.com/dokan-dev/dokany/releases/latest), let me know if you [figure out how](https://github.com/dokan-dev/dokany/wiki/FUSE)  
@@ -78,6 +78,7 @@ cd /mnt/nas/music/.hist
 # [`prisonparty.sh`](prisonparty.sh)
 * run copyparty in a chroot, preventing any accidental file access
 * creates bindmounts for /bin, /lib, and so on, see `sysdirs=`
+* **not compatible with:** `--th-bwrap`
 
 # [`bubbleparty.sh`](bubbleparty.sh)
 * run copyparty in an isolated process, preventing any accidental file access and more

bin/bubbleparty.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 # usage: ./bubbleparty.sh ./copyparty-sfx.py ....
 bwrap \
   --unshare-all \
@@ -9,7 +9,7 @@ bwrap \
   --dev-bind /dev /dev \
   --dir /tmp \
   --dir /var \
-  --bind $(pwd) $(pwd) \
+  --bind "$(pwd)" "$(pwd)" \
   --share-net \
   --die-with-parent \
   --file 11 /etc/passwd \

bin/dbtool.py

@@ -8,7 +8,7 @@ import sqlite3
 import argparse
 
 DB_VER1 = 3
-DB_VER2 = 5
+DB_VER2 = 6
 
 BY_PATH = None
 NC = None
@@ -39,7 +39,7 @@ def ls(db):
     print(f"{nfiles} files")
     print(f"{ntags} tags\n")
 
-    print("number of occurences for each tag,")
+    print("number of occurrences for each tag,")
     print(" 'x' = file has no tags")
     print(" 't:mtp' = the mtp flag (file not mtp processed yet)")
     print()

bin/handlers/404-to-fail2ban.py

@@ -0,0 +1,36 @@
+# /!\ Warning: be careful, as webdav clients often generate a large number of 404 requets.
+
+# In your `jail.local`, add:
+# [copyparty]
+# enabled = true
+# logtimezone = UTC
+# logpath = /path/to/log/file # or keep the default value if you're using systemd
+
+# Create the `copyparty.conf` file in `filter.d` with the following:
+# [Definition]
+# failregex = ^ <ADDR>$
+# ignoreregex =
+# datepattern = ^fail2ban: %%Y-%%m-%%dT%%H:%%M:%%S
+
+# First check `--dont-ban`, and if it doesn't match, log the line to be intercepted by fail2ban.
+from datetime import datetime, UTC
+def main(cli, vn="", rem=""):
+    now = datetime.now(UTC).isoformat()[:19]
+    msg = "\nfail2ban: %s %s"
+    if not vn and not rem:
+        # got called by --xban
+        cli["log"](msg % (now, cli["ip"]), 3)
+        return {"rc": 0}
+
+    cond = cli.args.dont_ban
+    if (
+        cond == "any"
+        or (cond == "auth" and cli.uname != "*")
+        or (cond == "aa" and cli.avol)
+        or (cond == "av" and cli.can_admin)
+        or (cond == "rw" and cli.can_read and cli.can_write)
+    ):
+        return ""
+
+    cli.log(msg % (now, cli.ip), 3)
+    return ""

bin/handlers/README.md

@@ -22,10 +22,11 @@ each plugin must define a `main()` which takes 3 arguments;
 
 * [redirect.py](redirect.py) sends an HTTP 301 or 302, redirecting the client to another page/file
 * [randpic.py](randpic.py) redirects `/foo/bar/randpic.jpg` to a random pic in `/foo/bar/`
-* [sorry.py](answer.py) replies with a custom message instead of the usual 404
+* [sorry.py](sorry.py) replies with a custom message instead of the usual 404
 * [nooo.py](nooo.py) replies with an endless noooooooooooooo
 * [never404.py](never404.py) 100% guarantee that 404 will never be a thing again as it automatically creates dummy files whenever necessary
 * [caching-proxy.py](caching-proxy.py) transforms copyparty into a squid/varnish knockoff
+* [404-to-fail2ban.py](404-to-fail2ban.py) creates 404 logs for fail2ban
 
 ## on403
 

bin/handlers/nooo.py

@@ -8,7 +8,7 @@ def say_no():
 
 
 def main(cli, vn, rem):
-    cli.send_headers(None, 404, "text/plain")
+    cli.send_headers("oh_f", None, 404, "text/plain")
 
     for chunk in say_no():
         cli.s.sendall(chunk)

bin/handlers/redirect.py

@@ -46,7 +46,7 @@ def main(cli, vn, rem):
 
     # uncomment one of these:
     send_http_302_temporary_redirect(cli, new_path)
-    #send_http_301_permanent_redirect(cli, new_path)
-    #send_errorpage_with_redirect_link(cli, new_path)
+    # send_http_301_permanent_redirect(cli, new_path)
+    # send_errorpage_with_redirect_link(cli, new_path)
 
     return "true"

bin/hooks/README.md

@@ -4,6 +4,11 @@ these programs either take zero arguments, or a filepath (the affected file), or
 
 run copyparty with `--help-hooks` for usage details / hook type explanations (xm/xbu/xau/xiu/xbc/xac/xbr/xar/xbd/xad/xban)
 
+in particular, if a hook is loaded into copyparty with the hook-flag `c` ("check") then its exit-code controls the action that launched the hook:
+* exit-code `0` = allow the action, and/or continue running the next hook
+* exit-code `100` = allow the action, and stop running any remaining consecutive hooks
+* anything else = reject/prevent the original action, and don't run the remaining hooks
+
 > **note:** in addition to event hooks (the stuff described here), copyparty has another api to run your programs/scripts while providing way more information such as audio tags / video codecs / etc and optionally daisychaining data between scripts in a processing pipeline; if that's what you want then see [mtp plugins](../mtag/) instead
 
 
@@ -28,10 +33,24 @@ these are `--xiu` hooks; unlike `xbu` and `xau` (which get executed on every sin
 * [reject-extension.py](reject-extension.py) rejects uploads if they match a list of file extensions
 * [reloc-by-ext.py](reloc-by-ext.py) redirects an upload to another destination based on the file extension
   * good example of the `reloc` [hook effect](https://github.com/9001/copyparty/blob/hovudstraum/docs/devnotes.md#hook-effects)
+* [reject-and-explain.py](reject-and-explain.py) shows a custom error-message when it rejects an upload
+* [reject-ramdisk.py](reject-ramdisk.py) rejects the upload if the destination is a ramdisk
+  * this hook uses the `I` flag which makes it 140x faster, but if the plugin has a bug it may crash copyparty
+* [phonecam-sorter.py](phonecam-sorter.py) is for syncing backups of phone gallery into year/month folders
+
+
+# more upload stuff
+* combine [reloc-by-wark-xbu.py](reloc-by-wark-xbu.py) and [reloc-by-wark-xau.py](reloc-by-wark-xau.py) to rename uploads to the checksum of the file contents
 
 
 # on message
 * [wget.py](wget.py) lets you download files by POSTing URLs to copyparty
+  * [wget-i.py](wget-i.py) is an import-safe modification of this hook (starts 140x faster, but higher chance of bugs)
 * [qbittorrent-magnet.py](qbittorrent-magnet.py) starts downloading a torrent if you post a magnet url
 * [usb-eject.py](usb-eject.py) adds web-UI buttons to safe-remove usb flashdrives shared through copyparty
 * [msg-log.py](msg-log.py) is a guestbook; logs messages to a doc in the same folder
+
+
+# general concept demos
+* [import-me.py](import-me.py) shows how the `I` flag makes the hook 140x faster (but you need to be Very Careful when writing the plugin)
+  * [wget-i.py](wget-i.py) is an import-safe modification of [wget.py](wget.py)

bin/hooks/import-me.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+
+from typing import Any
+
+_ = r"""
+the fastest hook in the west
+(runs directly inside copyparty, not as a subprocess)
+
+example usage as global config:
+    --xbu I,bin/hooks/import-me.py
+
+example usage as a volflag (per-volume config):
+    -v srv/inc:inc:r:rw,ed:c,xbu=I,bin/hooks/import-me.py
+                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+    (share filesystem-path srv/inc as volume /inc,
+     readable by everyone, read-write for user 'ed',
+     running this plugin on all uploads with the params listed below)
+
+example usage as a volflag in a copyparty config file:
+    [/inc]
+      srv/inc
+      accs:
+        r: *
+        rw: ed
+      flags:
+        xbu: I,bin/hooks/import-me.py
+
+parameters explained,
+    I = import; do not fork / subprocess
+
+IMPORTANT NOTE:
+    because this hook is running inside copyparty, you need to
+    be EXCEPTIONALLY CAREFUL to avoid side-effects, for example
+    DO NOT os.chdir() or anything like that, and also make sure
+    that the name of this file is unique (cannot be the same as
+    an existing python module/library)
+"""
+
+
+def main(ka: dict[str, Any]) -> dict[str, Any]:
+    # "ka" is a dictionary with info from copyparty...
+
+    # but because we are running inside copyparty, we don't need such courtesies;
+    import inspect
+
+    cf = inspect.currentframe().f_back.f_back.f_back
+    t = "hello from hook; I am able to peek into copyparty's memory like so:\n  function name: %s\n  variables:\n    %s\n"
+    t2 = "\n    ".join([("%r: %r" % (k, v))[:99] for k, v in cf.f_locals.items()][:9])
+    logger = ka["log"]
+    logger(t % (cf.f_code, t2))
+
+    # must return a dictionary with:
+    #  "rc": the retcode; 0 is ok
+    return {"rc": 0}

bin/hooks/notify.py

@@ -9,7 +9,7 @@ from plyer import notification
 _ = r"""
 show os notification on upload; works on windows, linux, macos, android
 
-depdencies:
+dependencies:
     windows: python3 -m pip install --user -U plyer
     linux:   python3 -m pip install --user -U plyer
     macos:   python3 -m pip install --user -U plyer pyobjus

bin/hooks/phonecam-sorter.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+
+import os
+import re
+import sys
+
+
+_ = r"""
+purpose:
+  sync/backup phone gallery (pics/vids) to a nas which runs copyparty
+
+what this hook does:
+  redirect photos/videos into date-based folders, named "%Y%m" according
+  to filename; that is, "/cam/sort/DCIM/D027/PXL_20250828_202820075.jpg"
+  will be redirected to "/cam/pics/ts/202508/PXL_20250828_202820075.jpg"
+
+requires this setup on the server:
+  * /cam/ is a volume with all your pics/vids
+  * the phone must upload to somewhere below /cam/sort/
+  * photo/video filenames match the following regex:
+     ^[a-z]{3,4}_([0-9]{6})[0-9]{2}_[0-9]{6}.*\.(jpg|mp4)$
+  * so for example these filenames are OK:
+    * PXL_20250828_202820075.jpg
+    * IMG_20251123_202405.jpg
+    * PANO_20180421_121245_03.t8o5.jpg
+    * VID_20250307_220250-crf34.mp4
+
+example server config as a volflag:
+  -v /mnt/thedisk/gallery:cam:rwa,ed:c,e2ds,xbu=I,~/hooks/phonecam-sorter.py
+
+example server config in a copyparty config file:
+  [/cam]
+    /mnt/thedisk/gallery
+    accs:
+      rwa: ed
+    flags:
+      e2ds
+      xbu: I,~/hooks/phonecam-sorter.py
+
+command to run on phone to sync to server, sending new files only:
+  ~/bin/u2c.py -te homenas.ca -a '$homenas.pw' https://10.0.0.2:3923/cam/sort/ ~/storage/dcim/Camera
+
+the phone-command above assumes this setup on the phone:
+* termux or another linux-like thing with access to gallery files
+* homenas.ca is a textfile with the server's CA certificate
+* homenas.pw is a textfile with the copyparty password inside
+* ~/storage/dcim/ exists; if not, run termux-setup-storage
+* ~/bin/u2c.py exists; if not, curl -LO https://10.0.0.2:3923/.cpr/a/u2c.py && chmod 755 u2c.py
+
+unfortunate consequences:
+* due to the redirection, u2c -z doesn't do anything, so every local
+   file is hashed every time, wasting some time and cpu on the phone
+   but otoh certainly avoids corruption on either side (phone/server)
+
+"""
+
+
+def main(inf):
+    log = inf["log"]
+    vdir, fn = os.path.split(inf["vp"])
+    if not vdir.startswith("cam/sort"):
+        return {"rc": 0}  # 2nd call after 1st reloc; already done
+
+    m = re.match(r"^[a-z]{3,4}_([0-9]{6})[0-9]{2}_[0-9]{6}.*\.(jpg|mp4)$", fn.lower())
+    if not m:
+        return {"rc": 0}  # not recognized; write to original path in /cam/sort/
+
+    ym, ext = m.groups()
+    if ext == "jpg":
+        dst = f"/cam/pics/ts/{ym}/"
+    elif ext == "mp4":
+        dst = f"/cam/vids/ts/{ym}/"
+    else:
+        return {"rc": 0}
+
+    ap = "%s%s" % (inf["ap"].split("/cam/sort/")[0], dst)
+    if not os.path.isdir(ap):
+        os.makedirs(ap)
+
+    ret = {"rc": 0, "reloc": {"vp": dst}}
+    return ret
+
+
+if __name__ == "__main__":
+    main()

bin/hooks/reject-and-explain.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+
+import json
+import os
+import re
+import sys
+
+
+_ = r"""
+reject file upload (with a nice explanation why)
+
+example usage as global config:
+    --xbu j,c1,bin/hooks/reject-and-explain.py
+
+example usage as a volflag (per-volume config):
+    -v srv/inc:inc:r:rw,ed:c,xbu=j,c1,bin/hooks/reject-and-explain.py
+                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+    (share filesystem-path srv/inc as volume /inc,
+     readable by everyone, read-write for user 'ed',
+     running this plugin on all uploads with the params listed below)
+
+example usage as a volflag in a copyparty config file:
+    [/inc]
+      srv/inc
+      accs:
+        r: *
+        rw: ed
+      flags:
+        xbu: j,c1,bin/hooks/reject-and-explain.py
+
+parameters explained,
+    xbu = execute-before-upload  (can also be xau, execute-after-upload)
+    j   = this hook needs upload information as json (not just the filename)
+    c1  = this hook returns json on stdout, so tell copyparty to read that
+"""
+
+
+def main():
+    inf = json.loads(sys.argv[1])
+    vdir, fn = os.path.split(inf["vp"])
+    print("inf[vp] = %r" % (inf["vp"],), file=sys.stderr)
+
+    # the following is what decides if we'll accept the upload or reject it:
+    # we check if the upload-folder url matches the following regex-pattern:
+    ok = re.search(r"(^|/)day[0-9]+$", vdir, re.IGNORECASE)
+
+    if ok:
+        # allow the upload
+        print("{}")
+        return
+
+    # the upload was rejected; display the following errortext:
+    errmsg = "Files can only be uploaded into a folder named 'DayN' where N is a number, for example 'Day573'. This file was REJECTED: "
+    errmsg += inf["vp"]  # if you want to mention the file's url
+    print(json.dumps({"rejectmsg": errmsg}))
+
+
+if __name__ == "__main__":
+    main()

bin/hooks/reject-ramdisk.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+
+import os
+import threading
+from argparse import Namespace
+
+from jinja2.nodes import Name
+from copyparty.fsutil import Fstab
+from typing import Any, Optional
+
+
+_ = r"""
+reject an upload if the target folder is on a ramdisk; useful when you
+have a volume where some folders inside are ramdisks but others aren't
+
+example usage as global config:
+    --xbu I,bin/hooks/reject-ramdisk.py
+
+example usage as a volflag (per-volume config):
+    -v srv/inc:inc:r:rw,ed:c,xbu=I,bin/hooks/reject-ramdisk.py
+                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+    (share filesystem-path srv/inc as volume /inc,
+     readable by everyone, read-write for user 'ed',
+     running this plugin on all uploads with the params listed below)
+
+example usage as a volflag in a copyparty config file:
+    [/inc]
+      srv/inc
+      accs:
+        r: *
+        rw: ed
+      flags:
+        xbu: I,bin/hooks/reject-ramdisk.py
+
+parameters explained,
+    I = import; do not fork / subprocess
+
+IMPORTANT NOTE:
+    because this hook is imported inside copyparty, you need to
+    be EXCEPTIONALLY CAREFUL to avoid side-effects, for example
+    DO NOT os.chdir() or anything like that, and also make sure
+    that the name of this file is unique (cannot be the same as
+    an existing python module/library)
+"""
+
+
+mutex = threading.Lock()
+fstab: Optional[Fstab] = None
+
+
+def main(ka: dict[str, Any]) -> dict[str, Any]:
+    global fstab
+    with mutex:
+        log = ka["log"]  # this is a copyparty NamedLogger function
+        if not fstab:
+            log("<HOOK:RAMDISK> creating fstab", 6)
+            args = Namespace()
+            args.mtab_age = 1  # cache the filesystem info for 1 sec
+            fstab = Fstab(log, args, False)
+
+        ap = ka["ap"]  # abspath the upload is going to
+        fs, mp = fstab.get(ap)  # figure out what the filesystem is
+        ramdisk = fs in ("tmpfs", "overlay")  # looks like a ramdisk?
+
+        # log("<HOOK:RAMDISK> fs=%r" % (fs,))
+
+        if ramdisk:
+            t = "Upload REJECTED because destination is a ramdisk"
+            return {"rc": 1, "rejectmsg": t}
+
+        return {"rc": 0}

bin/hooks/reloc-by-wark-xau.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+
+
+_ = r"""
+rename incoming uploads according to the "wark" (the file identifier)
+which is basically but not exactly a sha512 hash of the file contents
+
+NOTE: this does NOT work with up2k uploads (dragdrop into browser);
+  combine this hook with reloc-by-wark-xbu.py to fix that
+
+example usage as global config:
+    -e2d --xau I,c,bin/hooks/reloc-by-wark-xau.py
+
+parameters explained,
+    e2d = enable up2k database (mandatory for xau hooks)
+    xau = execute before upload
+    I   = import this hook for performance; do not fork / subprocess
+    c   = "check"; reject upload if this hook crashes due to a bug
+
+example usage as a volflag (per-volume config):
+    -v srv/inc:inc:r:rw,ed:c,e2d,xau=I,c,bin/hooks/reloc-by-wark-xau.py
+                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+    (share filesystem-path srv/inc as volume /inc,
+     readable by everyone, read-write for user 'ed',
+     running this plugin on all uploads with the params explained above)
+
+example usage as a volflag in a copyparty config file:
+    [/inc]
+      srv/inc
+      accs:
+        r: *
+        rw: ed
+      flags:
+        e2d, xau: I,c,bin/hooks/reloc-by-wark-xau.py
+"""
+
+
+def main(inf):
+    if inf.get("wark"):
+        # this is an up2k upload; nothing to be done, just bail
+        return {}
+
+    abspath = inf["ap"]  # filesystem path to the uploaded file
+
+    # we don't have the wark yet so need to calculate it;
+    # generating a regular sha512 would of course be much easier,
+    # but then filenames would be different depending on how the
+    # file was uploaded (laaame) so let's do it the hard way
+
+    # use the standard up2k-salt which nobody ever changes:
+    salt = "hunter2"
+
+    # to generate the wark we'll need some functions from copyparty;
+    # follow the trail to the copyparty module and grab them from there:
+
+    import inspect
+
+    libpath = inspect.getfile(inf["log"])
+    libpath = os.path.dirname(os.path.dirname(libpath))
+    sys.path.insert(0, libpath)
+
+    from copyparty.up2k import up2k_hashlist_from_file, up2k_wark_from_hashlist
+
+    chunklist, st = up2k_hashlist_from_file(abspath)
+    wark = up2k_wark_from_hashlist(salt, st.st_size, chunklist)
+
+    # okay nice
+    # the rest of the code below is just copied from reloc-by-wark-xbu.py
+    # -------------------------------------------------------------------------
+
+    # grab the original filename from the vpath...
+    vdir, fn = os.path.split(inf["vp"])
+
+    # ...to retain the original file extension, if any
+    try:
+        fn, ext = fn.rsplit(".", 1)
+    except:
+        ext = ""
+
+    # use the first 16 characters; 12 bytes of entropy,
+    # roughly one collision for every 26 million files
+    fn = wark[:16]
+
+    if ext:
+        ext = ext.lower()
+        fn += "." + ext
+
+    return {"reloc": {"fn": fn}}

bin/hooks/reloc-by-wark-xbu.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+
+import os
+
+
+_ = r"""
+rename incoming uploads according to the "wark" (the file identifier)
+which is basically but not exactly a sha512 hash of the file contents
+
+NOTE: this only works for up2k uploads (dragdrop into browser);
+  combine this with reloc-by-wark-xau.py to cover the other protocols
+
+example usage as global config:
+    -e2d --xbu I,c,bin/hooks/reloc-by-wark-xbu.py
+
+parameters explained,
+    e2d = enable up2k database (mandatory for xbu hooks)
+    xbu = execute before upload
+    I   = import this hook for performance; do not fork / subprocess
+    c   = "check"; reject upload if this hook crashes due to a bug
+
+example usage as a volflag (per-volume config):
+    -v srv/inc:inc:r:rw,ed:c,e2d,xbu=I,c,bin/hooks/reloc-by-wark-xbu.py
+                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+    (share filesystem-path srv/inc as volume /inc,
+     readable by everyone, read-write for user 'ed',
+     running this plugin on all uploads with the params explained above)
+
+example usage as a volflag in a copyparty config file:
+    [/inc]
+      srv/inc
+      accs:
+        r: *
+        rw: ed
+      flags:
+        e2d, xbu: I,c,bin/hooks/reloc-by-wark-xbu.py
+"""
+
+
+def main(inf):
+    wark = inf.get("wark")
+    if not wark:
+        # not an up2k upload, so we don't have the hash;
+
+        # option 1: let upload proceed with original filename
+        return {}
+
+        # option 2: reject the upload
+        return {"rejectmsg": "only up2k uploads are allowed in this volume"}
+
+    # grab the original filename from the vpath...
+    vdir, fn = os.path.split(inf["vp"])
+
+    # ...to retain the original file extension, if any
+    try:
+        fn, ext = fn.rsplit(".", 1)
+    except:
+        ext = ""
+
+    # use the first 16 characters; 12 bytes of entropy,
+    # roughly one collision for every 26 million files
+    fn = wark[:16]
+
+    if ext:
+        ext = ext.lower()
+        fn += "." + ext
+
+    return {"reloc": {"fn": fn}}

bin/hooks/usb-eject.py

@@ -34,7 +34,10 @@ MOUNT_BASE = b"/run/media/egon/"
 
 def main():
     try:
-        label = sys.argv[1].split(":usb-eject:")[1].split(":")[0]
+        msg = sys.argv[1]
+        if msg.startswith("upload-queue-empty;"):
+            return
+        label = msg.split(":usb-eject:")[1].split(":")[0]
         mp = MOUNT_BASE + unquote(label)
         # print("ejecting [%s]... " % (mp,), end="")
         mp = os.path.abspath(os.path.realpath(mp))

bin/hooks/wget-i.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+
+import os
+import threading
+import subprocess as sp
+
+
+_ = r"""
+use copyparty as a file downloader by POSTing URLs as
+application/x-www-form-urlencoded (for example using the
+📟 message-to-server-log in the web-ui)
+
+this hook is a modified copy of wget.py, modified to
+make it import-safe so it can be run with the 'I' flag,
+which speeds up the startup time of the hook by 140x
+
+example usage as global config:
+    --xm aw,I,bin/hooks/wget-i.py
+
+parameters explained,
+    xm = execute on message-to-server-log
+    aw = only users with write-access can use this
+    I = import; do not fork / subprocess
+
+example usage as a volflag (per-volume config):
+    -v srv/inc:inc:r:rw,ed:c,xm=aw,I,bin/hooks/wget.py
+                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+    (share filesystem-path srv/inc as volume /inc,
+     readable by everyone, read-write for user 'ed',
+     running this plugin on all messages with the params explained above)
+
+example usage as a volflag in a copyparty config file:
+    [/inc]
+      srv/inc
+      accs:
+        r: *
+        rw: ed
+      flags:
+        xm: aw,I,bin/hooks/wget.py
+
+the volflag examples only kicks in if you send the message
+while you're in the /inc folder (or any folder below there)
+
+IMPORTANT NOTE:
+    because this hook uses the 'I' flag to run inside copyparty,
+    many other flags will not work (f,j,c3,t3600 as seen in the
+    original wget.py), and furthermore + more importantly we
+    need to be EXCEPTIONALLY CAREFUL to avoid side-effects, so
+    the os.chdir has been replaced with cwd=dirpath for example
+"""
+
+
+def do_stuff(inf):
+    """
+    worker function which is executed in another thread to
+    avoid blocking copyparty while the download is running,
+    since we cannot use the 'f,t3600' hook-flags with 'I'
+    """
+
+    # first things first; grab the logger-function which copyparty is letting us borrow
+    log = inf["log"]
+
+    url = inf["txt"]
+    if url.startswith("upload-queue-empty;"):
+        return
+
+    if "://" not in url:
+        url = "https://" + url
+
+    proto = url.split("://")[0].lower()
+    if proto not in ("http", "https", "ftp", "ftps"):
+        raise Exception("bad proto {}".format(proto))
+
+    dirpath = inf["ap"]
+
+    name = url.split("?")[0].split("/")[-1]
+    msg = "-- DOWNLOADING " + name
+    log(msg)
+    tfn = os.path.join(dirpath, msg)
+    open(tfn, "wb").close()
+
+    cmd = ["wget", "--trust-server-names", "-nv", "--", url]
+
+    try:
+        # two things to note here:
+        # - cannot use the `c3` hook-flag with `I` so mute output with stdout=sp.DEVNULL instead;
+        # - MUST NOT use os.chdir with 'I' so use cwd=dirpath instead 
+        sp.check_call(cmd, cwd=dirpath, stdout=sp.DEVNULL)
+    except:
+        t = "-- FAILED TO DOWNLOAD " + name
+        log(t, 3)  # 3=yellow=warning
+        open(os.path.join(dirpath, t), "wb").close()
+        raise  # have copyparty scream about the details in the log
+
+    os.unlink(tfn)
+
+
+def main(inf):
+    threading.Thread(target=do_stuff, args=(inf,), daemon=True).start()

bin/hooks/wget.py

@@ -47,6 +47,9 @@ while you're in the /inc folder (or any folder below there)
 def main():
     inf = json.loads(sys.argv[1])
     url = inf["txt"]
+    if url.startswith("upload-queue-empty;"):
+        return
+
     if "://" not in url:
         url = "https://" + url
 
@@ -66,7 +69,7 @@ def main():
     try:
         sp.check_call(cmd)
     except:
-        t = "-- FAILED TO DONWLOAD " + name
+        t = "-- FAILED TO DOWNLOAD " + name
         print(f"{t}\n", end="")
         open(t, "wb").close()
 

bin/mtag/README.md

@@ -68,3 +68,8 @@ instead of affecting all volumes, you can set the options for just one volume li
 * `:c,mtp=key=f,audio-key.py`
 * `:c,mtp=.bpm=f,audio-bpm.py`
 * `:c,mtp=ahash,vhash=f,media-hash.py`
+
+
+# tips & tricks
+
+* to delete tags for all files below `blog*` and rescan that, `sqlite3 .hist/up2k.db "delete from mt where w in (select substr(w,1,16) from up where rd like 'blog%')";`

bin/mtag/audio-key.py

@@ -4,13 +4,19 @@ import os
 import sys
 import tempfile
 import subprocess as sp
-import keyfinder
+
+try:
+    import keyfinder
+
+    PKF = True
+except:
+    PKF = False
 
 from copyparty.util import fsenc
 
 """
 dep: github/mixxxdj/libkeyfinder
-dep: pypi/keyfinder
+dep: pypi/keyfinder  -OR-  EvanPurkhiser/keyfinder-cli
 dep: ffmpeg
 """
 
@@ -35,7 +41,17 @@ def det(tf):
     ])
     # fmt: on
 
-    print(keyfinder.key(tf).camelot())
+    if PKF:
+        print(keyfinder.key(tf).camelot())
+    else:
+        # fmt: off
+        sp.check_call([
+            b"keyfinder-cli",
+            b"-n",
+            b"camelot",
+            fsenc(tf)
+        ])
+        # fmt: on
 
 
 def main():

bin/mtag/geotag.py

@@ -0,0 +1,53 @@
+import json
+import re
+import sys
+
+from copyparty.util import fsenc, runcmd
+
+
+"""
+uses exiftool to geotag images based on embedded gps coordinates in exif data
+
+adds four new metadata keys:
+    .gps_lat = latitute
+    .gps_lon = longitude
+    .masl = meters above sea level
+    city = "city, subregion, region"
+
+usage: -mtp .masl,.gps_lat,.gps_lon,city=ad,t10,bin/mtag/geotag.py
+
+example: https://a.ocv.me/pub/blog/j7/8/?grid=0
+"""
+
+
+def main():
+    cmd = b"exiftool -api geolocation -n".split(b" ")
+    rc, so, se = runcmd(cmd + [fsenc(sys.argv[1])])
+    ptn = re.compile("([^:]*[^ :]) *: (.*)")
+    city = ["", "", ""]
+    ret = {}
+    for ln in so.split("\n"):
+        m = ptn.match(ln)
+        if not m:
+            continue
+        k, v = m.groups()
+        if k == "Geolocation City":
+            city[2] = v
+        elif k == "Geolocation Subregion":
+            city[1] = v
+        elif k == "Geolocation Region":
+            city[0] = v
+        elif k == "GPS Latitude":
+            ret[".gps_lat"] = "%.04f" % (float(v),)
+        elif k == "GPS Longitude":
+            ret[".gps_lon"] = "%.04f" % (float(v),)
+        elif k == "GPS Altitude":
+            ret[".masl"] = str(int(float(v)))
+    v = ", ".join(city).strip(", ")
+    if v:
+        ret["city"] = v
+    print(json.dumps(ret))
+
+
+if __name__ == "__main__":
+    main()

bin/mtag/guestbook-read.py

@@ -7,7 +7,7 @@ example copyparty config to use this:
   --urlform save,get -vsrv/hello:hello:w:c,e2ts,mtp=guestbook=t10,ad,p,bin/mtag/guestbook-read.py:mte=+guestbook
 
 explained:
-  for realpath srv/hello (served at /hello), write-only for eveyrone,
+  for realpath srv/hello (served at /hello), write-only for everyone,
   enable file analysis on upload (e2ts),
   use mtp plugin "bin/mtag/guestbook-read.py" to provide metadata tag "guestbook",
   do this on all uploads regardless of extension,

bin/mtag/guestbook.py

@@ -11,7 +11,7 @@ example copyparty config to use this:
   --urlform save,get -vsrv/hello:hello:w:c,e2ts,mtp=xgb=ebin,t10,ad,p,bin/mtag/guestbook.py:mte=+xgb
 
 explained:
-  for realpath srv/hello (served at /hello),write-only for eveyrone,
+  for realpath srv/hello (served at /hello),write-only for everyone,
   enable file analysis on upload (e2ts),
   use mtp plugin "bin/mtag/guestbook.py" to provide metadata tag "xgb",
   do this on all uploads with the file extension "bin",

bin/mtag/install-deps.sh

@@ -155,6 +155,11 @@ install_keyfinder() {
 		return
 	}
 
+	(cat /etc/alpine-release || echo a) 2>&1 | grep -E '3\.2[3-9]' && {
+		echo "alpine too new; ffmpeg8 is keyfinder-py incompat; giving up"
+		return
+	}
+
 	cd "$td"
 	github_tarball https://api.github.com/repos/mixxxdj/libkeyfinder/releases/latest
 	ls -al
@@ -189,7 +194,7 @@ install_keyfinder() {
 		exit 1
 	}
 
-	x=${-//[^x]/}; set -x; cat /etc/alpine-release
+	x=${-//[^x]/}; set -x; cat /etc/alpine-release || true
 	# rm -rf /Users/ed/Library/Python/3.9/lib/python/site-packages/*keyfinder*
 	CFLAGS="-I$h/pe/keyfinder/include -I/opt/local/include -I/usr/include/ffmpeg" \
 	CXXFLAGS="-I$h/pe/keyfinder/include -I/opt/local/include -I/usr/include/ffmpeg" \

bin/mtag/wget.py

@@ -84,7 +84,7 @@ def main():
         #   on success, delete the .bin file which contains the URL
         os.unlink(fp)
     except:
-        open("-- FAILED TO DONWLOAD " + name, "wb").close()
+        open("-- FAILED TO DOWNLOAD " + name, "wb").close()
 
     os.unlink(tfn)
     print(url)

bin/partyfuse.py

@@ -6,8 +6,8 @@ __copyright__ = 2019
 __license__ = "MIT"
 __url__ = "https://github.com/9001/copyparty/"
 
-S_VERSION = "2.0"
-S_BUILD_DT = "2024-10-01"
+S_VERSION = "2.2"
+S_BUILD_DT = "2025-12-16"
 
 """
 mount a copyparty server (local or remote) as a filesystem
@@ -21,7 +21,7 @@ usage:
   python partyfuse.py http://192.168.1.69:3923/  ./music
 
 dependencies:
-  python3 -m pip install --user fusepy  # or grab it from the connect page
+  python3 -m pip install --user mfusepy  # or grab it from the connect page
   + on Linux: sudo apk add fuse
   + on Macos: https://osxfuse.github.io/
   + on Windows: https://github.com/billziss-gh/winfsp/releases/latest
@@ -92,22 +92,28 @@ is_dbg = False
 
 
 try:
-    from fuse import FUSE, FuseOSError, Operations
+    from mfusepy import FUSE, FuseOSError, Operations
 except:
+    try:
+        from fuse import FUSE, FuseOSError, Operations
+    except:
+        FUSE = None
+
     if WINDOWS:
         libfuse = "install https://github.com/billziss-gh/winfsp/releases/latest"
     elif MACOS:
         libfuse = "install https://osxfuse.github.io/"
     else:
-        libfuse = "apt install libfuse3-3\n    modprobe fuse"
+        libfuse = "apt install libfuse2\n    modprobe fuse"
 
     m = """\033[33m
-  could not import fuse; these may help:
-    {} -m pip install --user fusepy
+  could not import mfusepy; these may help:
+    {} -m pip install --user mfusepy
     {}
 \033[0m"""
-    print(m.format(sys.executable, libfuse))
-    raise
+    if not FUSE:
+        print(m.format(sys.executable, libfuse))
+        raise
 
 
 def termsafe(txt):
@@ -143,10 +149,10 @@ def fancy_log(fmt, *a):
 
 
 def register_wtf8():
-    def wtf8_enc(text):
+    def wtf8_enc(text, errors=""):
         return str(text).encode("utf-8", "surrogateescape"), len(text)
 
-    def wtf8_dec(binary):
+    def wtf8_dec(binary, errors=""):
         return bytes(binary).decode("utf-8", "surrogateescape"), len(binary)
 
     def wtf8_search(encoding_name):
@@ -284,8 +290,8 @@ class Gateway(object):
             if ar.td:
                 self.ssl_context = ssl._create_unverified_context()
             elif ar.te:
-                self.ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS)
-                self.ssl_context.load_verify_locations(ar.te)
+                self.ssl_context = ssl.create_default_context(cafile=ar.te)
+                self.ssl_context.check_hostname = ar.teh
 
         self.conns = {}
 
@@ -359,7 +365,7 @@ class Gateway(object):
     def sendreq(self, meth, path, headers, **kwargs):
         tid = get_tid()
         if self.password:
-            headers["Cookie"] = "=".join(["cppwd", self.password])
+            headers["PW"] = self.password
 
         try:
             c = self.getconn(tid)
@@ -620,7 +626,7 @@ class Gateway(object):
             "st_atime": ts,
             "st_mtime": ts,
             "st_ctime": ts,
-            "st_blocks": int((sz + 511) / 512),
+            "st_blocks": -int(-sz // 512),
         }
 
     def stat_file(self, ts, sz):
@@ -632,7 +638,7 @@ class Gateway(object):
             "st_atime": ts,
             "st_mtime": ts,
             "st_ctime": ts,
-            "st_blocks": int((sz + 511) / 512),
+            "st_blocks": -int(-sz // 512),
         }
 
 
@@ -902,9 +908,7 @@ class CPPF(Operations):
         return ret
 
     def _readdir(self, path, fh=None):
-        path = path.strip("/")
-        dbg("readdir %r [%s]", path, fh)
-
+        dbg("dircache miss")
         ret = self.gw.listdir(path)
         if not self.n_dircache:
             return ret
@@ -914,11 +918,17 @@ class CPPF(Operations):
             self.dircache.append(cn)
             self.clean_dircache()
 
-        # import pprint; pprint.pprint(ret)
         return ret
 
     def readdir(self, path, fh=None):
-        return [".", ".."] + list(self._readdir(path, fh))
+        dbg("readdir %r [%s]", path, fh)
+        path = path.strip("/")
+        cn = self.get_cached_dir(path)
+        if cn:
+            ret = cn.data
+        else:
+            ret = self._readdir(path, fh)
+        return [".", ".."] + list(ret)
 
     def read(self, path, length, offset, fh=None):
         req_max = 1024 * 1024 * 8
@@ -993,7 +1003,6 @@ class CPPF(Operations):
         if cn:
             dents = cn.data
         else:
-            dbg("cache miss")
             dents = self._readdir(dirpath)
 
         try:
@@ -1141,10 +1150,15 @@ def main():
     if WINDOWS:
         examples.append("http://192.168.1.69:3923/music/  M:")
 
+    epi = "example:" + ex_pre + ex_pre.join(examples)
+    epi += """\n
+NOTE: if server has --usernames enabled, then password is "username:password"
+"""
+
     ap = argparse.ArgumentParser(
         formatter_class=TheArgparseFormatter,
         description="mount a copyparty server as a local filesystem -- " + ver,
-        epilog="example:" + ex_pre + ex_pre.join(examples),
+        epilog=epi,
     )
     # fmt: off
     ap.add_argument("base_url", type=str, help="remote copyparty URL to mount")
@@ -1157,6 +1171,7 @@ def main():
 
     ap2 = ap.add_argument_group("https/TLS")
     ap2.add_argument("-te", metavar="PEMFILE", help="certificate to expect/verify")
+    ap2.add_argument("-teh", action="store_true", help="require correct hostname in -te cert")
     ap2.add_argument("-td", action="store_true", help="disable certificate check")
 
     ap2 = ap.add_argument_group("cache/perf")

bin/prisonparty.sh

@@ -20,6 +20,8 @@ example:
 example for running straight from source (instead of using an sfx):
   PYTHONPATH=$PWD ./prisonparty.sh /var/lib/copyparty-jail cpp cpp /mnt/nas/music -- python3 -um copyparty -v /mnt/nas/music::rwmd
 
+NOT COMPATIBLE WITH: --th-bwrap
+
 note that if you have python modules installed as --user (such as bpm/key detectors),
   you should add /home/foo/.local as a VOLDIR
 
@@ -97,6 +99,7 @@ jail="${jail%/}"
 
 
 # bind-mount system directories and volumes
+mkdir -p "$jail"
 for a in {1..30}; do mkdir "$jail/.prisonlock" && break; sleep 0.1; done
 printf '%s\n' "${sysdirs[@]}" "${vols[@]}" | sed -r 's`/$``' | LC_ALL=C sort | uniq |
 while IFS= read -r v; do
@@ -141,7 +144,7 @@ chmod 777 "$jail/tmp"
 
 
 # create a dev
-(cd $jail; mkdir -p dev; cd dev
+(cd "$jail"; mkdir -p dev; cd dev
 [ -e null ]    || mknod -m 666 null    c 1 3
 [ -e zero ]    || mknod -m 666 zero    c 1 5
 [ -e random ]  || mknod -m 444 random  c 1 8

bin/thumbs/README.md

@@ -0,0 +1,55 @@
+# thumbnail extractors
+
+extract (or generate) thumbnails for custom file formats or override default thumbnail extraction for standard media files
+
+
+## usage
+
+* load plugins with `--th-extract baz=~/dev/copyparty/bin/thumbs/randomcolor.py`
+  * `baz` is the file extension (may be comma-separated list of extensions)
+  * add multiple different extractor plugins by repeating the `--th-extract` argument
+
+
+## api
+
+### in
+
+each plugin must define a function `main(abspath, **kwargs)`:
+* `abspath` is the path to a file to extract thumbnail for
+* `kwargs` currently receives these keyword arguments:
+  * `vn` is the VFS which contains the requested file
+  * `th_srv` is an instance of [copyparty/th_srv](https://github.com/9001/copyparty/blob/hovudstraum/copyparty/th_srv.py)
+
+### out
+
+the `main()` function must
+* return `None` or raise an `Exception` in case it is unable to extract thumbnail
+* return tuple `fmt, stream, offset, whence, size` otherwise:
+  * `fmt: str` – format (filename extension) of the thumbnail; should be one of the image formats supported by copyparty (see `--th-r-*` options)
+  * `stream: IO[bytes]` – binary file-like object supporting `seek()`, `read()`, `close()` methods; should contain extracted thumbnail image
+  * `offset: int, whence: int` – arguments to `stream.seek()`; copyparty will call `stream.seek(offset, whence)` once before starting reading the stream contents
+  * `size: int | None` – number of bytes to read from the `stream`; if `size` is negative or `None`, copyparty will read until the end of the stream
+
+
+### notes
+
+it is possible to extract or generate custom thumbnails for standard media files too,
+e.g. run copyparty with `--th-extract mp3,m4a,aac,flac,opus=~/dev/copyparty/bin/thumbs/my_custom_mp3_th_extractor.py` and do your magic with audio file thumbnails
+
+in case copyparty doesn't like the `fmt` option, it will `close()` the `stream` without ever calling `seek()` or `read()`
+
+if your scenario involves heavy computation or i/o, it is better to defer the heavy parts until the first `seek()` or `read()` to avoid redundant work in case the result gets rejected based on `fmt` value
+
+if you provide custom `stream` object:
+copyparty performs buffered reading, so expect multiple `read()` calls, respect `size` argument in `read(size)` calls, and maintain correct seek position
+
+
+## examples
+
+* [randomcolor](https://github.com/9001/copyparty/blob/hovudstraum/bin/thumbs/randomcolor.py) generates a random .gif image
+* [collabora](https://github.com/9001/copyparty/blob/hovudstraum/bin/thumbs/collabora.py) uses external self-hosted service (collabora online) to generate previews for various office documents, pdf and fb2 books, html, markdown, text files and some image formats; collabora is also supported as office editor in copyparty, see [wopi](https://github.com/9001/copyparty#wopi-server)
+
+
+## some other known plugins seen on the internets
+
+* [fpkg_thumb](https://github.com/kamaeff/fpkg-vault/blob/1c248de62a9ff9c4088d4b3e7962023cde9d6426/old/fpkg_thumb.py) extracts cover images from playstation4 software installation packages ("pkg" and "fpkg" files)

bin/thumbs/collabora.py

@@ -0,0 +1,125 @@
+"""
+CAUTION!
+looks like collabora spawns separate jailed libreoffice instance to do the conversion
+this may eat up too much ram in folders with many docs or with --th-pregen enabled
+
+consider limiting resources available for collabora
+e.g. for me these lines in docker-compose.yml helped alot:
+    deploy:
+      resources:
+        limits:
+          cpus: '2'
+          memory: '2G'
+/CAUTION!
+
+if you use collabora office editor with copyparty, you already have a thumbnailer for office formats:
+https://sdk.collaboraonline.com/docs/conversion_api.html
+the 'convert-to' api accepts files in multipart/form-data format
+
+just add this extractor to copyparty with an argument like this:
+  --th-extract=docx,xlsx,pptx,odt,ods,pdf,fb2=~/dev/copyparty/bin/thumbs/collabora.py
+and set COLLABORA_INSTANCE_URL below to your collabora instace url
+
+full list of supported file types:
+https://sdk.collaboraonline.com/docs/conversion_api.html#supported-input-formats
+
+this is also an example of deferred IO:
+no files opened and no network requests made if copyparty doesn't like the file type
+"""
+
+import os
+import string
+import random
+import mimetypes
+from urllib.error import HTTPError
+from urllib.request import Request, urlopen
+
+COLLABORA_INSTANCE_URL = "http://collabora:9980"
+BOUNDARY_CHARS = string.ascii_letters + string.digits
+
+
+def main(abspath, **kwargs):
+    boundary = "CopypartyFormData" + "".join(random.choices(BOUNDARY_CHARS, k=53))
+    start, end = make_headers(abspath, boundary)
+    size = len(start) + os.path.getsize(abspath) + len(end)
+    request = Request(
+        COLLABORA_INSTANCE_URL + "/cool/convert-to/png",
+        method="POST",
+        data=iterate(start, abspath, end),
+        headers={
+            "Content-Type": f"multipart/form-data; boundary={boundary}",
+            "Content-Length": str(size),
+        },
+    )
+
+    file_like = Wrapper(request)
+    offset, whence, length = 0, 0, None
+
+    return "png", file_like, offset, whence, length
+
+
+def make_headers(abspath, boundary):
+    """stuff to wrap file contents by for multipart"""
+
+    fname = "file." + abspath.rsplit(".", 1)[-1]
+    ftype = mimetypes.guess_file_type(fname)[0] or "application/octet-stream"
+    start = (
+        f"--{boundary}\r\n"
+        f'Content-Disposition: form-data; name="data"; filename="{fname}"\r\n'
+        f"Content-Type: {ftype}\r\n\r\n"
+    )
+    end = f"\r\n--{boundary}--"
+    return bytes(start, "ascii"), bytes(end, "ascii")
+
+
+def iterate(start, fpath, end):
+    """docs may be big so read and send them in chunks"""
+
+    yield start
+    with open(fpath, "rb") as f:
+        while True:
+            b = f.read(32768)
+            if not b:
+                break
+            yield b
+    yield end
+
+
+class Wrapper:
+    """
+    This file-like wrapper defers expensive work until first .seek() call
+    Useful  to not waste resources in case copyparty decides
+    to throw our thumbnail away based on returned file extension
+    """
+
+    def __init__(self, request):
+        self.request = request
+        self.response = None
+
+    def seek(self, offset, whence=0):
+        if self.response:
+            return 0
+
+        try:
+            # the heavy part:
+            # - reading probably big file
+            # - sending it over the network
+            # - making collabora do some work
+            self.response = urlopen(self.request, timeout=10)
+        except HTTPError as e:
+            e.close()
+            raise e
+        except Exception as e:
+            self.close()
+            raise e
+
+        # actual response is not seekable
+        # just return 0
+        return 0
+
+    def read(self, amount):
+        return self.response.read(amount)
+
+    def close(self):
+        if self.response:
+            self.response.close()

bin/thumbs/randomcolor.py

@@ -0,0 +1,29 @@
+import io
+import os
+import struct
+from base64 import b64decode
+
+
+def create_gif() -> bytes:
+    head = r"R0lGODdhIAAgAIAB"
+    tail = r"AAD/LAAAAAAgACAAAAIehI+py+0Po5y02ouz3rz7D4biSJbmiabqyrbuC5sFADs="
+    #color = struct.pack(">L", random.randrange(0xffffff))  # wow, way too stable
+    color = b"\x00" + os.urandom(3)
+    return b64decode(head) + color + b64decode(tail)
+
+
+def main(abspath, **ka) -> Optional[tuple[str, IO[bytes], int, int, Optional[int]]]:
+    print("hello from randomcolor.py pretending to extract a thumb from " + abspath)
+
+    # need to return a file-like object
+    gif_bytes = create_gif()
+    file_obj = io.BytesIO(gif_bytes)
+
+    # where copyparty should seek to in the "file" to start reading from
+    seek_offset = 0
+    seek_whence = 0
+
+    filesize = None  # auto; read until end
+
+    # "gif" is the file-extension to decode the file as
+    return "gif", file_obj, seek_offset, seek_whence, filesize

bin/u2c.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python3
-from __future__ import print_function, unicode_literals
+from __future__ import division, print_function, unicode_literals
 
-S_VERSION = "2.11"
-S_BUILD_DT = "2025-05-18"
+S_VERSION = "2.25"
+S_BUILD_DT = "2026-10-05"
 
 """
 u2c.py: upload to copyparty
@@ -10,7 +10,7 @@ u2c.py: upload to copyparty
 https://github.com/9001/copyparty/blob/hovudstraum/bin/u2c.py
 
 - dependencies: no
-- supports python 2.6, 2.7, and 3.3 through 3.12
+- supports python 2.6, 2.7, and 3.3 through 3.14
 - if something breaks just try again and it'll autoresume
 """
 
@@ -67,6 +67,8 @@ else:
 
 WTF8 = "replace" if PY2 else "surrogateescape"
 
+SEP = "{0}".format(os.sep).encode("ascii")
+
 VT100 = platform.system() != "Windows"
 
 
@@ -100,7 +102,7 @@ except:
     ub64enc = base64.urlsafe_b64encode
 
 
-class BadAuth(Exception):
+class Fatal(Exception):
     pass
 
 
@@ -165,8 +167,8 @@ class HCli(object):
             elif self.verify is True:
                 self.ctx = None
             else:
-                self.ctx = ssl.SSLContext(ssl.PROTOCOL_TLS)
-                self.ctx.load_verify_locations(self.verify)
+                self.ctx = ssl.create_default_context(cafile=self.verify)
+                self.ctx.check_hostname = ar.teh
 
         self.base_hdrs = {
             "Accept": "*/*",
@@ -196,6 +198,8 @@ class HCli(object):
         hdrs.update(self.base_hdrs)
         if self.ar.a:
             hdrs["PW"] = self.ar.a
+        if self.ar.ba:
+            hdrs["Authorization"] = self.ar.ba
         if ctype:
             hdrs["Content-Type"] = ctype
         if meth == "POST" and CLEN not in hdrs:
@@ -232,6 +236,7 @@ class HCli(object):
 
 MJ = "application/json"
 MO = "application/octet-stream"
+MM = "application/x-www-form-urlencoded"
 CLEN = "Content-Length"
 
 web = None  # type: HCli
@@ -246,7 +251,7 @@ class File(object):
 
     def __init__(self, top, rel, size, lmod):
         self.top = top  # type: bytes
-        self.rel = rel.replace(b"\\", b"/")  # type: bytes
+        self.rel = rel.replace(SEP, b"/")  # type: bytes
         self.size = size  # type: int
         self.lmod = lmod  # type: float
 
@@ -264,6 +269,7 @@ class File(object):
         self.wark = ""  # type: str
         self.url = ""  # type: str
         self.nhs = 0  # type: int
+        self.t_hs = 0.0  # type: float
 
         # set by upload
         self.t0_up = 0.0  # type: float
@@ -395,7 +401,7 @@ class MTHash(object):
             self.csz = chunksz
 
             chunks = {}
-            nchunks = int(math.ceil(fsz / chunksz))
+            nchunks = -int(-fsz // chunksz)
             for nch in range(nchunks):
                 self.work_q.put(nch)
 
@@ -491,6 +497,12 @@ print = safe_print if VT100 else flushing_print
 
 
 def termsize():
+    try:
+        w, h = os.get_terminal_size()
+        return w, h
+    except:
+        pass
+
     env = os.environ
 
     def ioctl_GWINSZ(fd):
@@ -590,9 +602,10 @@ def undns(url):
 
 def _scd(err, top):
     """non-recursive listing of directory contents, along with stat() info"""
+    top_ = os.path.join(top, b"")
     with os.scandir(top) as dh:
         for fh in dh:
-            abspath = os.path.join(top, fh.name)
+            abspath = top_ + fh.name
             try:
                 yield [abspath, fh.stat()]
             except Exception as ex:
@@ -601,8 +614,9 @@ def _scd(err, top):
 
 def _lsd(err, top):
     """non-recursive listing of directory contents, along with stat() info"""
+    top_ = os.path.join(top, b"")
     for name in os.listdir(top):
-        abspath = os.path.join(top, name)
+        abspath = top_ + name
         try:
             yield [abspath, os.stat(abspath)]
         except Exception as ex:
@@ -641,7 +655,7 @@ def walkdir(err, top, excl, seen):
 
 def walkdirs(err, tops, excl):
     """recursive statdir for a list of tops, yields [top, relpath, stat]"""
-    sep = "{0}".format(os.sep).encode("ascii")
+    sep = SEP
     if not VT100:
         excl = excl.replace("/", r"\\")
         za = []
@@ -677,7 +691,7 @@ def walkdirs(err, tops, excl):
                 yield stop, ap[len(stop) :].lstrip(sep), inf
         else:
             d, n = top.rsplit(sep, 1)
-            yield d, n, os.stat(top)
+            yield d or b"/", n, os.stat(top)
 
 
 # mostly from copyparty/util.py
@@ -714,7 +728,7 @@ def up2k_chunksize(filesize):
     stepsize = 512 * 1024
     while True:
         for mul in [1, 2]:
-            nchunks = math.ceil(filesize * 1.0 / chunksize)
+            nchunks = -int(-filesize // chunksize)
             if nchunks <= 256 or (chunksize >= 32 * 1024 * 1024 and nchunks <= 4096):
                 return chunksize
 
@@ -722,16 +736,22 @@ def up2k_chunksize(filesize):
             stepsize *= mul
 
 
-# mostly from copyparty/up2k.py
 def get_hashlist(file, pcb, mth):
     # type: (File, Any, Any) -> None
+    with open(file.abs, "rb", 512 * 1024) as f:
+        get_hashlist_f(file, pcb, mth, f)
+
+
+# mostly from copyparty/up2k.py
+def get_hashlist_f(file, pcb, mth, f):
+    # type: (File, Any, Any, Any) -> None
     """generates the up2k hashlist from file contents, inserts it into `file`"""
 
     chunk_sz = up2k_chunksize(file.size)
     file_rem = file.size
     file_ofs = 0
     ret = []
-    with open(file.abs, "rb", 512 * 1024) as f:
+    if True:
         t0 = time.time()
 
         if mth and file.size >= 1024 * 512:
@@ -767,6 +787,18 @@ def get_hashlist(file, pcb, mth):
             file.kchunks[k] = [v1, v2]
 
 
+def wark_stdin(ar):
+    file = File(b"", b"", int(ar.files[0]), 0)
+    get_hashlist_f(file, None, False, sys.stdin if PY2 else sys.stdin.buffer)
+    if ar.chs:
+        zsl = ["%s %d %d" % (zsii[0], n, zsii[1]) for n, zsii in enumerate(file.cids)]
+        print("chs:\n" + "\n".join(zsl))
+    zsl = [ar.wsalt, str(file.size)] + [x[0] for x in file.cids]
+    zb = hashlib.sha512("\n".join(zsl).encode("utf-8")).digest()[:33]
+    wark = ub64enc(zb).decode("utf-8")
+    print(wark)
+
+
 def printlink(ar, purl, name, fk):
     if not name:
         url = purl  # srch
@@ -824,10 +856,15 @@ def handshake(ar, file, search):
             url = ""
         url = ar.vtop + url
 
+    t0 = time.time()
+    tmax = t0 + ar.t_hs
     while True:
         sc = 600
         txt = ""
-        t0 = time.time()
+        t1 = time.time()
+        if t1 >= tmax:
+            print("\nERROR: server offline for longer than --t-hs; giving up")
+            raise Fatal()
         try:
             zs = json.dumps(req, separators=(",\n", ": "))
             sc, txt = web.req("POST", url, {}, zs.encode("utf-8"), MJ)
@@ -848,13 +885,13 @@ def handshake(ar, file, search):
                 return [], False
             elif sc == 409 or "<pre>upload rejected, file already exists" in txt:
                 return [], False
-            elif sc == 403:
+            elif sc == 403 or sc == 401:
                 print("\nERROR: login required, or wrong password:\n%s" % (txt,))
-                raise BadAuth()
+                raise Fatal()
 
-            t = "handshake failed, retrying: %s\n  t0=%.3f t1=%.3f td=%.3f\n  %s\n\n"
+            t = "handshake failed, retrying: %s\n  t0=%.3f t1=%.3f t2=%.3f td1=%.3f td2=%.3f\n  %s\n\n"
             now = time.time()
-            eprint(t % (file.name, t0, now, now - t0, em))
+            eprint(t % (file.name, t0, t1, now, now - t0, now - t1, em))
             time.sleep(ar.cd)
 
     try:
@@ -870,6 +907,7 @@ def handshake(ar, file, search):
     file.url = quotep(r["purl"].encode("utf-8", WTF8)).decode("utf-8")
     file.name = r["name"]
     file.wark = r["wark"]
+    file.t_hs = time.time()
 
     if ar.uon and not r["hash"]:
         printlink(ar, file.url, r["name"], r.get("fk"))
@@ -977,6 +1015,7 @@ class Ctl(object):
         self.nfiles, self.nbytes = self.stats
         self.filegen = walkdirs([], ar.files, ar.x)
         self.recheck = []  # type: list[File]
+        self.last_file = None
 
         if ar.safe:
             self._safe()
@@ -992,7 +1031,7 @@ class Ctl(object):
             self.up_b = 0  # num bytes handled
             self.up_br = 0  # num bytes actually transferred
             self.uploader_busy = 0
-            self.serialized = False
+            self.oneconn = False
 
             self.t0 = time.time()
             self.t0_up = None
@@ -1013,6 +1052,11 @@ class Ctl(object):
 
             self._fancy()
 
+        file = self.last_file
+        if self.up_br and file:
+            zs = quotep(file.name.encode("utf-8", WTF8))
+            web.req("POST", file.url, {}, b"msg=upload-queue-empty;" + zs, MM)
+
         self.ok = not self.errs
 
     def _safe(self):
@@ -1034,7 +1078,7 @@ class Ctl(object):
                 print("  hs...")
                 try:
                     hs, _ = handshake(self.ar, file, search)
-                except BadAuth:
+                except Fatal:
                     sys.exit(1)
 
                 if search:
@@ -1162,7 +1206,7 @@ class Ctl(object):
             handshake(self.ar, file, False)
 
     def cleanup_vt100(self):
-        if VT100:
+        if VT100 and not self.ar.ns:
             ss.scroll_region(None)
         else:
             eprint("\033]9;4;0\033\\")
@@ -1173,14 +1217,15 @@ class Ctl(object):
 
     def hasher(self):
         ptn = re.compile(self.ar.x.encode("utf-8"), re.I) if self.ar.x else None
-        sep = "{0}".format(os.sep).encode("ascii")
+        usep = os.sep
+        sep = SEP
         prd = None
         ls = {}
         for top, rel, inf in self.filegen:
             isdir = stat.S_ISDIR(inf.st_mode)
             if self.ar.z or self.ar.drd:
                 rd = rel if isdir else os.path.dirname(rel)
-                srd = rd.decode("utf-8", "replace").replace("\\", "/").rstrip("/")
+                srd = rd.decode("utf-8", "replace").replace(usep, "/").rstrip("/")
                 if srd:
                     srd += "/"
                 if prd != rd:
@@ -1190,7 +1235,7 @@ class Ctl(object):
                         print("      ls ~{0}".format(srd))
                         zt = (
                             self.ar.vtop,
-                            quotep(rd.replace(b"\\", b"/")).decode("utf-8"),
+                            quotep(rd.replace(sep, b"/")).decode("utf-8"),
                         )
                         sc, txt = web.req("GET", "%s%s?ls&lt&dots" % zt, {})
                         if sc >= 400:
@@ -1223,9 +1268,7 @@ class Ctl(object):
                             while req:
                                 print("DELETING ~%s#%s" % (srd, len(req)))
                                 body = json.dumps(req).encode("utf-8")
-                                sc, txt = web.req(
-                                    "POST", self.ar.url + "?delete", {}, body, MJ
-                                )
+                                sc, txt = web.req("POST", "/?delete", {}, body, MJ)
                                 if sc == 413 and "json 2big" in txt:
                                     print(" (delete request too big; slicing...)")
                                     req = req[: len(req) // 2]
@@ -1291,7 +1334,8 @@ class Ctl(object):
                 if self.ar.jw:
                     print("%s  %s" % (wark, vp))
                 else:
-                    zd = datetime.datetime.fromtimestamp(max(0, file.lmod), UTC)
+                    tsdt = datetime.datetime.fromtimestamp
+                    zd = tsdt(max(0, min(2 << 36, file.lmod)), UTC)
                     dt = "%04d-%02d-%02d %02d:%02d:%02d" % (
                         zd.year,
                         zd.month,
@@ -1341,7 +1385,7 @@ class Ctl(object):
 
             try:
                 hs, sprs = handshake(self.ar, file, search)
-            except BadAuth:
+            except Fatal:
                 self.panik = 1
                 break
 
@@ -1364,10 +1408,10 @@ class Ctl(object):
                 self.recheck.append(file)
 
             with self.mutex:
-                if hs and not sprs and not self.serialized:
-                    t = "server filesystem does not support sparse files; serializing uploads\n"
+                if hs and not sprs and not self.oneconn:
+                    t = "server filesystem does not support sparse files; cannot parallelize uploads\n"
                     eprint(t)
-                    self.serialized = True
+                    self.oneconn = True
                     for _ in range(self.ar.j - 1):
                         self.q_upload.put(None)
                 if not hs:
@@ -1379,7 +1423,7 @@ class Ctl(object):
                     if not file.recheck:
                         self.up_done(file)
 
-                if hs and file.up_c:
+                if hs and file.up_c and not self.oneconn:
                     # some chunks failed
                     self.up_c -= len(hs)
                     file.up_c -= len(hs)
@@ -1437,6 +1481,7 @@ class Ctl(object):
                 self.q_upload.put(fsl)
 
     def uploader(self):
+        drop_hs = (0.0, "")
         while True:
             fsl = self.q_upload.get()
             if not fsl:
@@ -1453,6 +1498,10 @@ class Ctl(object):
 
             file = fsl.file
             cids = fsl.cids
+            self.last_file = file
+
+            if self.oneconn and drop_hs == (file.t_hs, file.wark):
+                continue
 
             with self.mutex:
                 if not self.uploader_busy:
@@ -1479,6 +1528,9 @@ class Ctl(object):
                 t = "upload failed, retrying: %s #%s+%d (%s)\n"
                 eprint(t % (file.name, cids[0][:8], len(cids) - 1, ex))
                 file.cd = time.time() + self.ar.cd
+                if self.oneconn:
+                    file.ucids = []
+                    drop_hs = (file.t_hs, file.wark)
                 # handshake will fix it
 
             with self.mutex:
@@ -1527,16 +1579,17 @@ def main():
 
     # fmt: off
     ap = app = argparse.ArgumentParser(formatter_class=APF, description="copyparty up2k uploader / filesearch tool  " + ver, epilog="""
-NOTE:
-source file/folder selection uses rsync syntax, meaning that:
+NOTE: source file/folder selection uses rsync syntax, meaning that:
   "foo" uploads the entire folder to URL/foo/
   "foo/" uploads the CONTENTS of the folder into URL/
+NOTE: if server has --usernames enabled, then password is "username:password"
 """)
 
     ap.add_argument("url", type=unicode, help="server url, including destination folder")
     ap.add_argument("files", type=files_decoder, nargs="+", help="files and/or folders to process")
     ap.add_argument("-v", action="store_true", help="verbose")
-    ap.add_argument("-a", metavar="PASSWD", help="password or $filepath")
+    ap.add_argument("-a", metavar="PASSWD", default="", help="password (or $filepath) for copyparty (is sent in header 'PW'), default is env.var U2C_PW")
+    ap.add_argument("--ba", metavar="PASSWD", default="", help="password (or $filepath) for basic-auth (usually not necessary)")
     ap.add_argument("-s", action="store_true", help="file-search (disables upload)")
     ap.add_argument("-x", type=unicode, metavar="REGEX", action="append", help="skip file if filesystem-abspath matches REGEX (option can be repeated), example: '.*/\\.hist/.*'")
     ap.add_argument("--ok", action="store_true", help="continue even if some local files are inaccessible")
@@ -1564,6 +1617,7 @@ source file/folder selection uses rsync syntax, meaning that:
     ap.add_argument("--wsalt", type=unicode, metavar="S", default="hunter2", help="salt to use when creating warks; must match server config")
     ap.add_argument("--chs", action="store_true", help="verbose (print the hash/offset of each chunk in each file)")
     ap.add_argument("--jw", action="store_true", help="just identifier+filepath, not mtime/size too")
+    ap.add_argument("--stdin", action="store_true", help="calculate file-ID of stdin; u2c.py --stdin - $LEN < a.mkv")
 
     ap = app.add_argument_group("performance tweaks")
     ap.add_argument("-j", type=int, metavar="CONNS", default=2, help="parallel connections")
@@ -1574,11 +1628,13 @@ source file/folder selection uses rsync syntax, meaning that:
     ap.add_argument("-ns", action="store_true", help="no status panel (for slow consoles and macos)")
     ap.add_argument("--cxp", type=float, metavar="SEC", default=57, help="assume http connections expired after SEConds")
     ap.add_argument("--cd", type=float, metavar="SEC", default=5, help="delay before reattempting a failed handshake/upload")
+    ap.add_argument("--t-hs", type=float, metavar="SEC", default=186, help="crash if handshakes fail due to server-offline for this long")
     ap.add_argument("--safe", action="store_true", help="use simple fallback approach")
     ap.add_argument("-z", action="store_true", help="ZOOMIN' (skip uploading files if they exist at the destination with the ~same last-modified timestamp, so same as yolo / turbo with date-chk but even faster)")
 
     ap = app.add_argument_group("tls")
     ap.add_argument("-te", metavar="PATH", help="path to ca.pem or cert.pem to expect/verify")
+    ap.add_argument("-teh", action="store_true", help="require correct hostname in -te cert")
     ap.add_argument("-td", action="store_true", help="disable certificate check")
     # fmt: on
 
@@ -1592,6 +1648,10 @@ source file/folder selection uses rsync syntax, meaning that:
             except:
                 pass
 
+    if ar.stdin:
+        wark_stdin(ar)
+        return
+
     # msys2 doesn't uncygpath absolute paths with whitespace
     if not VT100:
         zsl = []
@@ -1674,11 +1734,18 @@ source file/folder selection uses rsync syntax, meaning that:
             print("\n\n   %s\n\n" % (t,))
             raise
 
-    if ar.a and ar.a.startswith("$"):
-        fn = ar.a[1:]
-        print("reading password from file [{0}]".format(fn))
-        with open(fn, "rb") as f:
-            ar.a = f.read().decode("utf-8").strip()
+    for k in ("a", "ba"):
+        zs = getattr(ar, k)
+        if zs.startswith("$"):
+            print("reading password from file [%s]" % (zs[1:],))
+            with open(zs[1:], "rb") as f:
+                setattr(ar, k, f.read().decode("utf-8").strip())
+
+    if not ar.a:
+        ar.a = os.environ.get("U2C_PW", "")
+
+    if ar.ba:
+        ar.ba = "Basic " + base64.b64encode(ar.ba.encode("utf-8")).decode("utf-8")
 
     for n in range(ar.rh):
         try:

bin/zmq-recv.py

@@ -61,6 +61,8 @@ def rep_server():
         print("copyparty says %r" % (sck.recv_string(),))
         reply = b"thx"
         # reply = b"return 1"  # non-zero to block an upload
+        # reply = b'{"rc":1}'  # or as json, that's fine too
+        # reply = b'{"rejectmsg":"naw dude"}'  # or custom message
         sck.send(reply)
 
 

contrib/README.md

@@ -7,6 +7,12 @@
 * works on windows, linux and macos
 * assumes `copyparty-sfx.py` was renamed to `copyparty.py` in the same folder as `copyparty.bat`
 
+### [`setup-ashell.sh`](setup-ashell.sh)
+* run copyparty on an iPhone/iPad using [a-Shell](https://holzschu.github.io/a-Shell_iOS/)
+* not very useful due to limitations in iOS:
+  * not able to share all of your phone's storage
+  * cannot run in the background
+
 ### [`index.html`](index.html)
 * drop-in redirect from an httpd to copyparty
 * assumes the webserver and copyparty is running on the same server/IP

contrib/haproxy/copyparty.conf

@@ -17,8 +17,12 @@ defaults
 
 listen foo1
   bind *:8081
+  http-request add-header X-Forwarded-Proto https if { ssl_fc }
+  http-request add-header X-Forwarded-Proto http unless { ssl_fc }
   server srv1 127.0.0.1:3923 maxconn 512
 
 listen foo2
   bind *:8082
+  http-request add-header X-Forwarded-Proto https if { ssl_fc }
+  http-request add-header X-Forwarded-Proto http unless { ssl_fc }
   server srv1 /dev/shm/party.sock maxconn 512

contrib/nginx/copyparty.conf

@@ -6,7 +6,7 @@
 #
 # to reverse-proxy a specific path/subpath/location below a domain
 # (rather than a complete subdomain), for example "/qw/er", you must
-# run copyparty with --rp-loc /qw/as and also change the following:
+# run copyparty with --rp-loc /qw/er and also change the following:
 # 	location / {
 # 		proxy_pass http://cpp_tcp;
 # to this:
@@ -31,7 +31,7 @@
 # generate the list of permitted IP ranges like so:
 #   (curl -s https://www.cloudflare.com/ips-v{4,6} | sed 's/^/allow /; s/$/;/'; echo; echo "deny all;") > /etc/nginx/cloudflare-only.conf
 #
-# and then enable it below by uncomenting the cloudflare-only.conf line
+# and then enable it below by uncommenting the cloudflare-only.conf line
 #
 # ======================================================================
 
@@ -75,9 +75,8 @@ server {
 		# recommendation: replace cpp_tcp with cpp_uds below
 		proxy_pass http://cpp_tcp;
 		proxy_redirect off;
-		# disable buffering (next 4 lines)
+		# disable buffering (next 3 lines)
 		proxy_http_version 1.1;
-		client_max_body_size 0;
 		proxy_buffering off;
 		proxy_request_buffering off;
 		# improve download speed from 600 to 1500 MiB/s

contrib/nixos/modules/copyparty.nix

@@ -48,9 +48,13 @@ let
 
   accountsWithPlaceholders = mapAttrs (name: attrs: passwordPlaceholder name);
 
+  volumesWithoutVariables = filterAttrs (k: v: !(hasInfix "\${" v.path)) cfg.volumes;
+
   configStr = ''
     ${mkSection "global" cfg.settings}
+    ${cfg.globalExtraConfig}
     ${mkSection "accounts" (accountsWithPlaceholders cfg.accounts)}
+    ${mkSection "groups" cfg.groups}
     ${concatStringsSep "\n" (mapAttrsToList mkVolume cfg.volumes)}
   '';
 
@@ -65,11 +69,8 @@ in
   options.services.copyparty = {
     enable = mkEnableOption "web-based file manager";
 
-    package = mkOption {
-      type = types.package;
-      default = pkgs.copyparty;
-      defaultText = "pkgs.copyparty";
-      description = ''
+    package = mkPackageOption pkgs "copyparty" {
+      extraDescription = ''
         Package of the application to run, exposed for overriding purposes.
       '';
     };
@@ -78,7 +79,7 @@ in
       type = types.bool;
       default = true;
       description = ''
-        Make a shell script wrapper called 'copyparty-hash' with all options set here,
+        Make a shell script wrapper called {command}`copyparty-hash` with all options set here,
         that launches the hashing cli.
       '';
     };
@@ -113,9 +114,9 @@ in
       type = types.attrs;
       description = ''
         Global settings to apply.
-        Directly maps to values in the [global] section of the copyparty config.
+        Directly maps to values in the `[global]` section of the copyparty config.
         Cannot set "c" or "hist", those are set by this module.
-        See `${getExe cfg.package} --help` for more details.
+        See {command}`copyparty --help` for more details.
       '';
       default = {
         i = "127.0.0.1";
@@ -131,6 +132,12 @@ in
       '';
     };
 
+    globalExtraConfig = mkOption {
+      type = types.str;
+      default = "";
+      description = "Appended to the end of the `[global]` section verbatim. This is useful for flags which are used in a repeating manner (e.g. `ipu: 255.255.255.1=user`) which can't be repeated in the settings = {} attribute set.";
+    };
+
     accounts = mkOption {
       type = types.attrsOf (
         types.submodule (
@@ -160,6 +167,19 @@ in
       '';
     };
 
+    groups = mkOption {
+      type = types.attrsOf (types.listOf types.str);
+      description = ''
+        A set of copyparty groups to create and the users that should be part of each group.
+      '';
+      default = { };
+      example = literalExpression ''
+        {
+          group_name = [ "user1" "user2" ];
+        };
+      '';
+    };
+
     volumes = mkOption {
       type = types.attrsOf (
         types.submodule (
@@ -178,21 +198,21 @@ in
                   Attribute list of permissions and the users to apply them to.
 
                   The key must be a string containing any combination of allowed permission:
-                    "r" (read):   list folder contents, download files
-                    "w" (write):  upload files; need "r" to see the uploads
-                    "m" (move):   move files and folders; need "w" at destination
-                    "d" (delete): permanently delete files and folders
-                    "g" (get):    download files, but cannot see folder contents
-                    "G" (upget):  "get", but can see filekeys of their own uploads
-                    "h" (html):   "get", but folders return their index.html
-                    "a" (admin):  can see uploader IPs, config-reload
+                  * "r" (read):   list folder contents, download files
+                  * "w" (write):  upload files; need "r" to see the uploads
+                  * "m" (move):   move files and folders; need "w" at destination
+                  * "d" (delete): permanently delete files and folders
+                  * "g" (get):    download files, but cannot see folder contents
+                  * "G" (upget):  "get", but can see filekeys of their own uploads
+                  * "h" (html):   "get", but folders return their index.html
+                  * "a" (admin):  can see uploader IPs, config-reload
 
                   For example: "rwmd"
 
                   The value must be one of:
-                    an account name, defined in `accounts`
-                    a list of account names
-                    "*", which means "any account"
+                  * an account name, defined in `accounts`
+                  * a list of account names
+                  * "*", which means "any account"
                 '';
                 example = literalExpression ''
                   {
@@ -207,7 +227,7 @@ in
                 type = types.attrs;
                 description = ''
                   Attribute list of volume flags to apply.
-                  See `${getExe cfg.package} --help-flags` for more details.
+                  See {command}`copyparty --help-flags` for more details.
                 '';
                 example = literalExpression ''
                   {
@@ -265,6 +285,7 @@ in
 
         environment = {
           PYTHONUNBUFFERED = "true";
+          NIX_SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
           XDG_CONFIG_HOME = externalStateDir;
         };
 
@@ -304,7 +325,7 @@ in
           BindPaths =
             (if cfg.settings ? hist then [ cfg.settings.hist ] else [ ])
             ++ [ externalStateDir ]
-            ++ (mapAttrsToList (k: v: v.path) cfg.volumes);
+            ++ (mapAttrsToList (k: v: v.path) volumesWithoutVariables);
           # ProtectSystem = "strict";
           # Note that unlike what 'ro' implies,
           # this actually makes it impossible to read anything in the root FS,
@@ -343,18 +364,32 @@ in
               #: in front of things means it wont change it if the directory already exists.
               group = ":${cfg.group}";
               user = ":${cfg.user}";
-              mode = ":755";
+              mode = ":${
+                # Use volume permissions if set
+                if (value.flags ? chmod_d) then
+                  value.flags.chmod_d
+                # Else, use global permission if set
+                else if (cfg.settings ? chmod-d) then
+                  cfg.settings.chmod-d
+                # Else, use the default permission
+                else
+                  "755"
+              }";
             };
           }
-        ) cfg.volumes
+        ) volumesWithoutVariables
       );
 
-      users.groups.copyparty = lib.mkIf (cfg.user == "copyparty" && cfg.group == "copyparty") { };
-      users.users.copyparty = lib.mkIf (cfg.user == "copyparty" && cfg.group == "copyparty") {
-        description = "Service user for copyparty";
-        group = "copyparty";
-        home = externalStateDir;
-        isSystemUser = true;
+      users.groups = lib.mkIf (cfg.group == "copyparty") {
+        copyparty = { };
+      };
+      users.users = lib.mkIf (cfg.user == "copyparty") {
+        copyparty = {
+          description = "Service user for copyparty";
+          group = cfg.group;
+          home = externalStateDir;
+          isSystemUser = true;
+        };
       };
       environment.systemPackages = lib.mkIf cfg.mkHashWrapper [
         (pkgs.writeShellScriptBin "copyparty-hash" ''

contrib/openrc/copyparty

@@ -9,10 +9,19 @@
 # you may want to:
 #   change '/usr/bin/python' to another interpreter
 #   change '/mnt::rw' to another location or permission-set
+#   use a config file instead of command arguments, e.g.:
+#      command_args="-c /etc/copyparty.conf"
 
 name="$SVCNAME"
 command_background=true
+extra_commands="checkconfig"
 pidfile="/var/run/$SVCNAME.pid"
 
 command="/usr/bin/python3 /usr/local/bin/copyparty-sfx.py"
 command_args="-q -v /mnt::rw"
+
+checkconfig() {
+        ebegin "Checking $RC_SVCNAME configuration"
+        $command $command_args --exit cfg
+        eend $?
+}

contrib/package/arch/PKGBUILD

@@ -3,9 +3,9 @@
 # NOTE: You generally shouldn't use this PKGBUILD on Arch, as it is mainly for testing purposes. Install copyparty using pacman instead.
 
 pkgname=copyparty
-pkgver="1.19.1"
+pkgver="1.20.26"
 pkgrel=1
-pkgdesc="File server with accelerated resumable uploads, dedup, WebDAV, FTP, TFTP, zeroconf, media indexer, thumbnails++"
+pkgdesc="File server with accelerated resumable uploads, dedup, WebDAV, SFTP, FTP, TFTP, zeroconf, media indexer, thumbnails++"
 arch=("any")
 url="https://github.com/9001/${pkgname}"
 license=('MIT')
@@ -13,22 +13,22 @@ depends=("bash" "python" "lsof" "python-jinja")
 makedepends=("python-wheel" "python-setuptools" "python-build" "python-installer" "make" "pigz")
 optdepends=("ffmpeg: thumbnails for videos, images (slower) and audio, music tags"
             "cfssl: generate TLS certificates on startup"
-            "python-mutagen: music tags (alternative)"
+            "python-mutagen: music tags (fast/inaccurate)"
             "python-pillow: thumbnails for images"
-            "python-pyvips: thumbnails for images (higher quality, faster, uses more ram)"
+            "python-pyvips: thumbnails for images (eats ram, prefer pillow)"
+            "python-paramiko: sftp server"
+            "python-pyftpdlib: ftp server"
+            "python-pyopenssl: ftps server (needs pyftpdlib too)"
             "libkeyfinder: detection of musical keys"
-            "python-pyopenssl: ftps functionality"
             "python-pyzmq: send zeromq messages from event-hooks"
             "python-argon2-cffi: hashed passwords in config"
 )
 source=("https://github.com/9001/${pkgname}/releases/download/v${pkgver}/${pkgname}-${pkgver}.tar.gz")
-backup=("etc/${pkgname}/copyparty.conf" )
-sha256sums=("bbc250db23eb80bc96c27b2efa456ce1e7f49c7dfaabadb91a571f70064b6f91")
+backup=("etc/copyparty.conf")
+install=copyparty.install
+sha256sums=("ac7ea05bde8d431b32cc2ff618f2754e46bd967bf10deaa6654dac9a449a6ff9")
 
 build() {
-    cd "${srcdir}/${pkgname}-${pkgver}/copyparty/web"
-    make
-
     cd "${srcdir}/${pkgname}-${pkgver}"
     python -m build --wheel --no-isolation
 }
@@ -37,12 +37,11 @@ package() {
     cd "${srcdir}/${pkgname}-${pkgver}"
     python -m installer --destdir="$pkgdir" dist/*.whl
 
-    install -dm755 "${pkgdir}/etc/${pkgname}"
     install -Dm755 "bin/prisonparty.sh" "${pkgdir}/usr/bin/prisonparty"
-    install -Dm644 "contrib/systemd/${pkgname}.conf" "${pkgdir}/etc/${pkgname}/copyparty.conf"
+    install -Dm644 "contrib/systemd/${pkgname}.conf" "${pkgdir}/etc/copyparty.conf"
     install -Dm644 "contrib/systemd/${pkgname}@.service" "${pkgdir}/usr/lib/systemd/system/${pkgname}@.service"
     install -Dm644 "contrib/systemd/${pkgname}-user.service" "${pkgdir}/usr/lib/systemd/user/${pkgname}.service"
     install -Dm644 "contrib/systemd/prisonparty@.service" "${pkgdir}/usr/lib/systemd/system/prisonparty@.service"
-    install -Dm644 "contrib/systemd/index.md" "${pkgdir}/var/lib/${pkgname}-jail/README.md"
+    install -Dm644 "contrib/systemd/index.md" "${pkgdir}/var/lib/${pkgname}-jail/srv/README.md"
     install -Dm644 "LICENSE" "${pkgdir}/usr/share/licenses/${pkgname}/LICENSE"
 }

contrib/package/arch/copyparty.install

@@ -0,0 +1,27 @@
+post_upgrade() {
+
+    # v1.20.22 moved the config-file, affecting the system-service only
+    if [[ $(vercmp "$2" 1.20.22) -lt 0 ]]; then
+
+        oldpath="/etc/copyparty/copyparty.conf"
+        newpath="/etc/copyparty.conf"
+
+        echo "==> Migrating config-file from $oldpath to $newpath"
+
+        if [[ ! -f "$oldpath" ]]; then
+            oldpath="$oldpath.pacsave"
+        fi
+
+        if [[ -f "$newpath.pacnew" ]]; then
+            echo "==> WARNING: A config-file already existed in the new location [$newpath]. The existing file will be used as-is."
+
+        elif [[ -f "$oldpath" ]]; then
+            mv -v "$oldpath" "$newpath"
+            rmdir /etc/copyparty 2>/dev/null || true
+        fi
+
+        systemctl cat copyparty@.service | awk '/=PRTY_CONFIG=/ && !/=\/etc\/copyparty.conf/ {exit 1}' || {
+            echo "==> WARNING: The copyparty system-service has been edited by hand since installation, so it will not read config from the new default config path [$newpath]. Please use PRTY_CONFIG=$newpath if that is correct for your setup."
+        }
+    fi
+}

contrib/package/makedeb-mpr/PKGBUILD

@@ -2,9 +2,9 @@
 
 
 pkgname=copyparty
-pkgver=1.19.1
+pkgver=1.20.26
 pkgrel=1
-pkgdesc="File server with accelerated resumable uploads, dedup, WebDAV, FTP, TFTP, zeroconf, media indexer, thumbnails++"
+pkgdesc="File server with accelerated resumable uploads, dedup, WebDAV, SFTP, FTP, TFTP, zeroconf, media indexer, thumbnails++"
 arch=("any")
 url="https://github.com/9001/${pkgname}"
 license=('MIT')
@@ -13,6 +13,7 @@ makedepends=("python3-wheel" "python3-setuptools" "python3-build" "python3-insta
 optdepends=("ffmpeg: thumbnails for videos, images (slower) and audio, music tags"
             "golang-cfssl: generate TLS certificates on startup"
             "python3-mutagen: music tags (alternative)"
+            "python3-paramiko: sftp server"
             "python3-pil: thumbnails for images"
             "python3-openssl: ftps functionality"
             "python3-zmq: send zeromq messages from event-hooks"
@@ -20,12 +21,9 @@ optdepends=("ffmpeg: thumbnails for videos, images (slower) and audio, music tag
 )
 source=("https://github.com/9001/${pkgname}/releases/download/v${pkgver}/${pkgname}-${pkgver}.tar.gz")
 backup=("/etc/${pkgname}.d/init" )
-sha256sums=("bbc250db23eb80bc96c27b2efa456ce1e7f49c7dfaabadb91a571f70064b6f91")
+sha256sums=("ac7ea05bde8d431b32cc2ff618f2754e46bd967bf10deaa6654dac9a449a6ff9")
 
 build() {
-    cd "${srcdir}/${pkgname}-${pkgver}/copyparty/web"
-    make
-
     cd "${srcdir}/${pkgname}-${pkgver}"
     python -m build --wheel --no-isolation
 }

contrib/package/makedeb-mpr/copyparty.service

@@ -15,7 +15,7 @@ Description=copyparty file server
 Type=notify
 SyslogIdentifier=copyparty
 Environment=PYTHONUNBUFFERED=x
-WorkingDirectory=/var/lib/copyparty-jail
+WorkingDirectory=/var/lib/copyparty-jail/srv
 ExecReload=/bin/kill -s USR1 $MAINPID
 
 # user to run as + where the TLS certificate is (if any)
@@ -26,7 +26,7 @@ Environment=XDG_CONFIG_HOME=/home/cpp/.config
 ExecStartPre=+/bin/bash -c 'mkdir -p /run/tmpfiles.d/ && echo "x /tmp/pe-copyparty*" > /run/tmpfiles.d/copyparty.conf'
 
 # run copyparty
-ExecStart=/usr/bin/python3 /usr/bin/copyparty -c /etc/copyparty.d/init
+ExecStart=/usr/bin/python3 /usr/local/bin/copyparty -c /etc/copyparty.d/init
 
 [Install]
 WantedBy=multi-user.target

contrib/package/makedeb-mpr/index.md

@@ -1,3 +1,13 @@
-this is `/var/lib/copyparty-jail`, the fallback webroot when copyparty has not yet been configured
+this is `/var/lib/copyparty-jail/srv/`, the fallback webroot when copyparty has not yet been configured
 
-please add some `*.conf` files to `/etc/copyparty.d/`
+please edit the config-file and restart copyparty;
+* if running as a system service: `vim /etc/copyparty.conf`
+* if running as a user service: `vim $HOME/.config/copyparty/copyparty.conf`
+
+some inspiration:
+* a basic example config: https://github.com/9001/copyparty/blob/hovudstraum/contrib/systemd/copyparty.example.conf
+* another example with focus on syntax: https://github.com/9001/copyparty/blob/hovudstraum/docs/example.conf
+* and CTRL-F `config file example` here: https://github.com/9001/copyparty
+
+the full list of configuration options can be seen at https://copyparty.eu/cli/ 
+or by running `copyparty --help`

contrib/package/makedeb-mpr/prisonparty.service

@@ -17,7 +17,7 @@ Description=copyparty file server
 [Service]
 SyslogIdentifier=prisonparty
 Environment=PYTHONUNBUFFERED=x
-WorkingDirectory=/var/lib/copyparty-jail
+WorkingDirectory=/var/lib/copyparty-jail/srv
 ExecReload=/bin/kill -s USR1 $MAINPID
 
 # stop systemd-tmpfiles-clean.timer from deleting copyparty while it's running

contrib/package/nix/copyparty/default.nix

@@ -15,6 +15,7 @@
   pyzmq,
   ffmpeg,
   mutagen,
+  paramiko,
   pyftpdlib,
   magic,
   partftpy,
@@ -29,7 +30,7 @@
   # create thumbnails with Pillow; faster than FFmpeg / MediaProcessing
   withThumbnails ? true,
 
-  # create thumbnails with PyVIPS; even faster, uses more memory
+  # create thumbnails with PyVIPS; uses more memory, pillow is preferred
   # -- can be combined with Pillow to support more filetypes
   withFastThumbnails ? false,
 
@@ -44,6 +45,9 @@
   # send ZeroMQ messages from event-hooks
   withZeroMQ ? true,
 
+  # enable SFTP server
+  withSFTP ? false,
+
   # enable FTP server
   withFTP ? true,
 
@@ -67,37 +71,93 @@
   # additional dependencies
   extraPythonPackages ? (_p: [ ]),
 
+  # to build stable + unstable with the same file
+  stable ? true,
+
+  # for commit date, only used when stable = false
+  copypartyFlake ? null,
+
+  nix-gitignore,
 }:
 
 let
   pinData = lib.importJSON ./pin.json;
   runtimeDeps = ([ util-linux ] ++ extraPackages ++ lib.optional withMediaProcessing ffmpeg);
+  inherit (copypartyFlake) lastModifiedDate;
+  # ex: "1970" "01" "01"
+  dateStringsZeroPrefixed = {
+    year = builtins.substring 0 4 lastModifiedDate;
+    month = builtins.substring 4 2 lastModifiedDate;
+    day = builtins.substring 6 2 lastModifiedDate;
+  };
+  # ex: "1970" "1" "1"
+  dateStringsShort = builtins.mapAttrs (
+    _: val: toString (lib.toIntBase10 val)
+  ) dateStringsZeroPrefixed;
+  unstableVersion =
+    if copypartyFlake == null then
+      "${pinData.version}-unstable"
+    else
+      with dateStringsZeroPrefixed; "${pinData.version}-unstable-${year}-${month}-${day}";
+  version = if stable then pinData.version else unstableVersion;
+  stableSrc = fetchurl {
+    inherit (pinData) url hash;
+  };
+  root = ../../../..;
+  unstableSrc = nix-gitignore.gitignoreSource [ ] root;
+  src = if stable then stableSrc else unstableSrc;
+  rev = copypartyFlake.shortRev or copypartyFlake.dirtyShortRev or "unknown";
+  unstableCodename = "unstable" + (lib.optionalString (copypartyFlake != null) "-${rev}");
 in
 buildPythonApplication {
   pname = "copyparty";
-  inherit (pinData) version;
-  src = fetchurl {
-    inherit (pinData) url hash;
-  };
-  dependencies =
-    [
-      jinja2
-      fusepy
-    ]
-    ++ lib.optional withSMB impacket
-    ++ lib.optional withFTP pyftpdlib
-    ++ lib.optional withFTPS pyopenssl
-    ++ lib.optional withTFTP partftpy
-    ++ lib.optional withCertgen cfssl
-    ++ lib.optional withThumbnails pillow
-    ++ lib.optional withFastThumbnails pyvips
-    ++ lib.optional withMediaProcessing ffmpeg
-    ++ lib.optional withBasicAudioMetadata mutagen
-    ++ lib.optional withHashedPasswords argon2-cffi
-    ++ lib.optional withZeroMQ pyzmq
-    ++ lib.optional withMagic magic
-    ++ (extraPythonPackages python.pkgs);
-  makeWrapperArgs = [ "--prefix PATH : ${lib.makeBinPath runtimeDeps}" ];
+  inherit version src;
+  postPatch = lib.optionalString (!stable) ''
+    old_src="$(mktemp -d)"
+    tar -C "$old_src" -xf ${stableSrc}
+    declare -a folders
+    folders=("$old_src"/*)
+    count_folders="''${#folders[@]}"
+    if [[ $count_folders != 1 ]]; then
+      declare -p folders
+      echo "Expected 1 folder, found $count_folders" >&2
+      exit 1
+    fi
+    old_src_folder="''${folders[0]}"
+    cp -r "$old_src_folder"/copyparty/web/deps copyparty/web/deps
+    sed -i 's/^CODENAME =.*$/CODENAME = "${unstableCodename}"/' copyparty/__version__.py
+    ${lib.optionalString (copypartyFlake != null) (
+      with dateStringsShort;
+      ''
+        sed -i 's/^BUILD_DT =.*$/BUILD_DT = (${year}, ${month}, ${day})/' copyparty/__version__.py
+      ''
+    )}
+  '';
+  dependencies = [
+    jinja2
+    fusepy
+  ]
+  ++ lib.optional withSMB impacket
+  ++ lib.optional withSFTP paramiko
+  ++ lib.optional withFTP pyftpdlib
+  ++ lib.optional withFTPS pyopenssl
+  ++ lib.optional withTFTP partftpy
+  ++ lib.optional withCertgen cfssl
+  ++ lib.optional withThumbnails pillow
+  ++ lib.optional withFastThumbnails pyvips
+  ++ lib.optional withMediaProcessing ffmpeg
+  ++ lib.optional withBasicAudioMetadata mutagen
+  ++ lib.optional withHashedPasswords argon2-cffi
+  ++ lib.optional withZeroMQ pyzmq
+  ++ lib.optional withMagic magic
+  ++ (extraPythonPackages python.pkgs);
+
+  makeWrapperArgs = [
+    "--prefix"
+    "PATH"
+    ":"
+    (lib.makeBinPath runtimeDeps)
+  ];
 
   pyproject = true;
   build-system = [
@@ -106,7 +166,7 @@ buildPythonApplication {
   meta = {
     description = "Turn almost any device into a file server";
     longDescription = ''
-      Portable file server with accelerated resumable uploads, dedup, WebDAV,
+      Portable file server with accelerated resumable uploads, dedup, WebDAV, SFTP,
       FTP, TFTP, zeroconf, media indexer, thumbnails++ all in one file, no deps
     '';
     homepage = "https://github.com/9001/copyparty";

contrib/package/nix/copyparty/pin.json

@@ -1,5 +1,5 @@
 {
-    "url": "https://github.com/9001/copyparty/releases/download/v1.19.1/copyparty-1.19.1.tar.gz",
-    "version": "1.19.1",
-    "hash": "sha256-u8JQ2yPrgLyWwnsu+kVs4ef0nH36q625GlcfcAZLb5E="
+    "url": "https://github.com/9001/copyparty/releases/download/v1.20.26/copyparty-1.20.26.tar.gz",
+    "version": "1.20.26",
+    "hash": "sha256-rH6gW96NQxsyzC/2GPJ1Tka9lnvxDeqmZU2smkSab/k="
 }

contrib/package/nix/overlay.nix

@@ -0,0 +1,32 @@
+final: prev:
+let
+  fullAttrs = {
+    withHashedPasswords = true;
+    withCertgen = true;
+    withThumbnails = true;
+    withFastThumbnails = true;
+    withMediaProcessing = true;
+    withBasicAudioMetadata = true;
+    withZeroMQ = true;
+    withSFTP = true;
+    withFTP = true;
+    withFTPS = true;
+    withTFTP = true;
+    withSMB = true;
+    withMagic = true;
+  };
+
+  call = attrs: final.python3.pkgs.callPackage ./copyparty ({ ffmpeg = final.ffmpeg-headless; } // attrs);
+in
+{
+  copyparty = call { stable = true; };
+  copyparty-unstable = call { stable = false; };
+  copyparty-full = call (fullAttrs // { stable = true; });
+  copyparty-unstable-full = call (fullAttrs // { stable = false; });
+
+  python3 = prev.python3.override {
+    packageOverrides = pyFinal: pyPrev: {
+      partftpy = pyFinal.callPackage ./partftpy { };
+    };
+  };
+}

contrib/package/rpm/copyparty.spec

@@ -5,15 +5,15 @@ License:        MIT
 Group:          Utilities
 URL:            https://github.com/9001/copyparty
 Source0:        copyparty-$pkgver.tar.gz
-Summary:        File server with accelerated resumable uploads, dedup, WebDAV, FTP, TFTP, zeroconf, media indexer, thumbnails++
+Summary:        File server with accelerated resumable uploads, dedup, WebDAV, SFTP, FTP, TFTP, zeroconf, media indexer, thumbnails++
 BuildArch:      noarch
 BuildRequires:  python3, python3-devel, pyproject-rpm-macros, python-setuptools, python-wheel, make
 Requires:       python3, (python3-jinja2 or python-jinja2), lsof
 Recommends:     ffmpeg, (golang-github-cloudflare-cfssl or cfssl), python-mutagen, python-pillow, python-pyvips
-Recommends:     qm-vamp-plugins, python-argon2-cffi, (python-pyopenssl or pyopenssl), python-impacket
+Recommends:     qm-vamp-plugins, python-argon2-cffi, (python-pyopenssl or pyopenssl), python-paramiko, python-impacket
 
 %description
-Portable file server with accelerated resumable uploads, dedup, WebDAV, FTP, TFTP, zeroconf, media indexer, thumbnails++ all in one file, no deps
+Portable file server with accelerated resumable uploads, dedup, WebDAV, SFTP, FTP, TFTP, zeroconf, media indexer, thumbnails++ all in one file, no deps
 
 See release at https://github.com/9001/copyparty/releases
 
@@ -26,9 +26,6 @@ See release at https://github.com/9001/copyparty/releases
 %setup -q
 
 %build
-cd "copyparty/web"
-make
-cd -
 %pyproject_wheel
 
 %install

contrib/plugins/README.md

@@ -39,3 +39,9 @@ point `--css-browser` to one of these by URL:
 
 * turns copyparty into chromecast just more flexible (and probably way more buggy)
 * usage: put the js somewhere in the webroot and `--js-browser /memes/meadup.js`
+
+
+
+# junk
+
+* [**rave.js**](./rave.js): april-fools joke, [demo (epilepsy warning)](https://cd.ocv.me/b/d2/d21/#af-9b927c42,sorthref), not maintained, very buggy

contrib/plugins/graft-thumbs.js

@@ -36,7 +36,7 @@
 
 		for (var a = 0; a < files.length; a++) {
 			var file = files[a],
-				is_pic = /\.(jpe?g|png|gif|webp)$/i.exec(file.vp),
+				is_pic = /\.(jpe?g|png|gif|webp|jxl)$/i.exec(file.vp),
 				is_audio = re_au_all.exec(file.vp),
 				basename = file.vp.replace(/\.[^\.]+$/, ""),
 				entry = pairs[basename];

contrib/plugins/minimal-up2k.js

@@ -66,11 +66,14 @@ var u2min = `
 
 </style>
 
-<a href="#" onclick="this.parentNode.innerHTML='';">show advanced options</a>
+<a href="#" id="u2min_off">show advanced options</a>
 `;
 
 if (!has(perms, 'read')) {
   var e2 = mknod('div');
   e2.innerHTML = u2min;
   ebi('wrap').insertBefore(e2, QS('#wfp'));
+  ebi('u2min_off').onclick = function () {
+    this.parentNode.innerHTML='';
+  };
 }

contrib/plugins/quickmove.js

@@ -101,10 +101,10 @@
 
     function our_hotkey_handler(e) {
         // bail if either ALT, CTRL, or SHIFT is pressed
-        if (e.altKey || e.shiftKey || e.isComposing || ctrl(e))
+        if (anymod(e))
             return main_hotkey_handler(e);  // let copyparty handle this keystroke
 
-        var key_name = (e.code || e.key) + '',
+        var keycode = (e.key || e.code) + '',
     		ae = document.activeElement,
 		    aet = ae && ae != document.body ? ae.nodeName.toLowerCase() : '';
 
@@ -114,7 +114,7 @@
         if (aet && !/^(a|button|tr|td|div|pre)$/.test(aet))
             return main_hotkey_handler(e);  // let copyparty handle this keystroke
 
-        if (key_name == 'KeyW') {
+        if (keycode == 'w' || keycode == 'KeyW') {
             // okay, this one's for us... do the thing
             action_to_perform();
             return ev(e);

contrib/podman-systemd/README.md

@@ -0,0 +1,173 @@
+# copyparty with Podman and Systemd
+
+Use this configuration if you want to run copyparty in a Podman container, with the reliability of running the container under a systemd service.
+
+Documentation for `.container` files can be found in the [Container unit](https://docs.podman.io/en/latest/markdown/podman-systemd.unit.5.html#container-units-container) docs. Systemd does not understand `.container` files natively, so Podman converts these to `.service` files with a [systemd-generator](https://www.freedesktop.org/software/systemd/man/latest/systemd.generator.html). This process is transparent, but sometimes needs to be debugged in case your `.container` file is malformed. There are instructions to debug the systemd generator in the Troubleshooting section below.
+
+To run copyparty in this way, you must already have podman installed. To install Podman, see: https://podman.io/docs/installation
+
+There is a sample configuration file in the same directory as this file (`copyparty.conf`).
+
+## Run the container as root
+
+Running the container as the root user is easy to set up, but less secure. There are instructions in the next section to run the container as a rootless user if you'd rather run the container like that.
+
+First, change this line in the `copyparty.container` file to reflect the directory you want to share. By default, it shares `/mnt/` but you'll probably want to change that.
+
+```
+# Change /mnt to something you want to share
+Volume=/mnt:/w:z
+```
+
+Note that you can select the owner and group of this volume by changing the `uid:` and `gid:` of the volume in `copyparty.conf`, but for simplicity let's assume you want it to be owned by `root:root`.
+
+To install and start copyparty with Podman and systemd as the root user, run the following:
+
+```shell
+sudo mkdir -pv /etc/containers/systemd/ /etc/copyparty/
+sudo cp -v copyparty.container /etc/containers/systemd/
+sudo cp -v copyparty.conf /etc/copyparty/
+sudo systemctl daemon-reload
+sudo systemctl start copyparty
+```
+
+Note: You can't "enable" this kind of Podman service. The `[Install]` section of the `.container` file effectively handles enabling the service so that it starts when the server reboots.
+
+You can see the status of the service with:
+
+```shell
+sudo systemctl status -a copyparty
+```
+
+You can see (and follow) the logs with either of these commands:
+
+```shell
+sudo podman logs -f copyparty
+
+# -a is required or else you'll get output like: copyparty[549025]: [649B blob data]
+sudo journalctl -a -f -u copyparty
+```
+
+## Run the container as a non-root user
+
+This configuration is more secure, but is more involved and requires ensuring files have proper permissions. You will need a root user account to do some of this setup.
+
+First, you need a user to run the container as. In this example we'll create a "podman" user with UID=1001 and GID=1001.
+
+```shell
+sudo groupadd -g 1001 podman
+sudo useradd -u 1001 -m podman
+sudo usermod -aG podman podman
+sudo loginctl enable-linger podman
+# Set a strong password for this user
+sudo -u podman passwd
+```
+
+The `enable-linger` command allows the podman user to run systemd user services that persist even when the user is not logged in. You could use a user that already exists in the system to run this service as, just make sure to run `loginctl enable-linger USERNAME` for that user.
+
+Next, change these lines in the `copyparty.container` file to reflect the config directory and the directory you want to share. By default, the config shares `/home/podman/copyparty/sharing/` but you'll probably want to change this:
+
+```
+# Change to reflect your non-root user's home directory
+Volume=/home/podman/copyparty/config:/cfg:z
+
+# Change to the directory you want to share
+Volume=/home/podman/copyparty/sharing:/w:z
+```
+
+Make sure the podman user has read/write access to both of these directories.
+
+Next, **log in to the server as the podman user**.
+
+To install and start copyparty as the non-root podman user, run the following:
+
+```shell
+mkdir -pv /home/podman/.config/containers/systemd/ /home/podman/copyparty/config
+cp -v copyparty.container /home/podman/.config/containers/systemd/copyparty.container
+cp -v copyparty.conf /home/podman/copyparty/config
+systemctl --user daemon-reload
+systemctl --user start copyparty
+```
+
+**Important note: Never use `sudo` with `systemctl --user`!**
+
+You can check the status of the user service with:
+
+```shell
+systemctl --user status -a copyparty
+```
+
+You can see (and follow) the logs with:
+
+```shell
+podman logs -f copyparty
+
+journalctl --user -a -f -u copyparty
+```
+
+## Troubleshooting
+
+If the container fails to start, and you've modified the `.container` service, it's likely that your `.container` file failed to be translated into a `.service` file. You can debug the podman service generator with this command:
+
+```shell
+sudo /usr/lib/systemd/system-generators/podman-system-generator --dryrun
+```
+
+## Allowing Traffic from Outside your Server
+
+To allow traffic on port 3923 of your server, you should run:
+
+```shell
+sudo firewall-cmd --permanent --add-port=3923/tcp
+sudo firewall-cmd --reload
+```
+
+Otherwise, you won't be able to access the copyparty server from anywhere other than the server itself.
+
+## Updating copyparty
+
+To update the version of copyparty used in the container, you can:
+
+```shell
+# If root:
+sudo podman pull docker.io/copyparty/ac:latest
+sudo systemctl restart copyparty
+
+# If non-root:
+podman pull docker.io/copyparty/ac:latest
+systemctl --user restart copyparty
+```
+
+Or, you can change the pinned version of the image in the `[Container]` section of the `.container` file and run:
+
+```shell
+# If root:
+sudo systemctl daemon-reload
+sudo systemctl restart copyparty
+
+# If non-root:
+systemctl --user daemon-reload
+systemctl --user restart copyparty
+```
+
+Podman will pull the image you've specified when restarting. If you have it set to `:latest`, Podman does not know to re-pull the container.
+
+### Enabling auto-update
+
+Alternatively, you can enable auto-updates by un-commenting this line:
+
+```
+# AutoUpdate=registry
+```
+
+You will also need to enable the [podman auto-updater service](https://docs.podman.io/en/latest/markdown/podman-auto-update.1.html) with:
+
+```shell
+# If root:
+sudo systemctl enable podman-auto-update.timer podman-auto-update.service
+
+# If non-root:
+systemctl --user enable podman-auto-update.timer podman-auto-update.service
+```
+
+This works best if you always want the latest version of copyparty. The auto-updater runs once every 24 hours.

contrib/podman-systemd/copyparty.conf

@@ -0,0 +1,44 @@
+[global]
+  e2dsa  # enable file indexing and filesystem scanning
+  e2ts   # and enable multimedia indexing
+  ansi   # and colors in log messages
+
+  # uncomment the line starting with q, lo: to log to a file instead of stdout/journalctl;
+  # $LOGS_DIRECTORY is usually /var/log/copyparty (comes from systemd)
+  # and copyparty replaces %Y-%m%d with Year-MonthDay, so the
+  # full path will be something like /var/log/copyparty/2023-1130.txt
+  # (note: enable compression by adding .xz at the end)
+  # q, lo: ${LOGS_DIRECTORY}/%Y-%m%d.log
+
+  # p: 80,443,3923   # listen on 80/443 as well (requires CAP_NET_BIND_SERVICE)
+  # i: 127.0.0.1     # only allow connections from localhost (reverse-proxies)
+  # ftp: 3921        # enable ftp server on port 3921
+  # p: 3939          # listen on another port
+  # df: 16           # stop accepting uploads if less than 16 GB free disk space
+  # ver              # show copyparty version in the controlpanel
+  # grid             # show thumbnails/grid-view by default
+  # theme: 2         # monokai
+  # name: datasaver  # change the server-name that's displayed in the browser
+  # stats, nos-dup   # enable the prometheus endpoint, but disable the dupes counter (too slow)
+  # no-robots, force-js  # make it harder for search engines to read your server
+
+  # enable version-checking by uncommenting one of the vc-url lines below;
+  # shows a warning-banner in the controlpanel if your version has a known vulnerability
+  #vc-url: https://api.github.com/repos/9001/copyparty/security-advisories?per_page=9
+  #vc-url: https://api.copyparty.eu/advisories
+  vc-exit  # panic and shutdown instead of just showing the warning
+
+
+[accounts]
+  ed: wark  # username: password
+
+
+[/]            # create a volume at "/" (the webroot), which will
+  /w           # share the contents of the "/w" folder
+  accs:
+    rw: *      # everyone gets read-write access, but
+    rwmda: ed  # the user "ed" gets read-write-move-delete-admin
+  flags:
+    e2ds       # enable filesystem-scanning for this volume only
+    # uid: 1000  # If you're running as root, you can change the owner of this volume here
+    # gid: 1000  # If you're running as root, you can change the group of this volume here

contrib/podman-systemd/copyparty.container

@@ -0,0 +1,55 @@
+[Container]
+# It's recommended to replace :latest with a specific version
+# for example: docker.io/copyparty/ac:1.19.15
+Image=docker.io/copyparty/ac:latest
+ContainerName=copyparty
+
+# Uncomment to enable auto-updates
+# AutoUpdate=registry
+
+# Environment variables
+# enable mimalloc by replacing "NOPE" with "2" for a nice speed-boost (will use twice as much ram)
+Environment=LD_PRELOAD=/usr/lib/libmimalloc-secure.so.NOPE
+# ensures log-messages are not delayed (but can reduce speed a tiny bit)
+Environment=PYTHONUNBUFFERED=1
+
+# Ports
+PublishPort=3923:3923
+
+
+# Volumes (PLEASE LOOK!)
+
+# Rootful setup:
+#   Leave as-is
+# Non-root setup:
+#   Change /etc/copyparty to /home/<USER>/copyparty/config
+Volume=/etc/copyparty:/cfg:z
+
+# Rootful setup:
+#   Change /mnt to the directory you want to share
+# Non-root setup:
+#   Change /mnt to something owned by your user, e.g., /home/<USER>/copyparty/sharing:/w:z
+Volume=/mnt:/w:z
+
+
+# Give the container time to stop in case the thumbnailer is still running.
+# It's allowed to continue finishing up for 10s after the shutdown signal, give it a 5s buffer
+StopTimeout=15
+
+# hide it from logs with "/._" so it matches the default --lf-url filter
+HealthCmd="wget --spider -q 127.0.0.1:3923/?reset=/._"
+HealthInterval=1m
+HealthTimeout=2s
+HealthRetries=5
+HealthStartPeriod=15s
+
+[Unit]
+After=default.target
+
+[Install]
+# Start by default on boot
+WantedBy=default.target
+
+[Service]
+# Give the container time to start in case it needs to pull the image
+TimeoutStartSec=600

contrib/rc/copyparty

@@ -1,7 +1,7 @@
 #!/bin/sh
 #
 # PROVIDE: copyparty
-# REQUIRE: networking
+# REQUIRE: networking DAEMON FILESYSTEMS mountd
 # KEYWORD:
 
 . /etc/rc.subr
@@ -10,7 +10,7 @@ name="copyparty"
 rcvar="copyparty_enable"
 copyparty_user="copyparty"
 copyparty_args="-e2dsa -v /storage:/storage:r" # change as you see fit
-copyparty_command="/usr/local/bin/python3.9 /usr/local/copyparty/copyparty-sfx.py ${copyparty_args}"
+copyparty_command="/usr/local/bin/python3.11 /usr/local/copyparty/copyparty-sfx.py ${copyparty_args}"
 pidfile="/var/run/copyparty/${name}.pid"
 command="/usr/sbin/daemon"
 command_args="-P ${pidfile} -r -f ${copyparty_command}"

contrib/setup-ashell.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+#
+# this script will install copyparty onto an iOS device (iPhone/iPad)
+#
+# step 1: install a-Shell:
+#   https://apps.apple.com/us/app/a-shell/id1473805438
+#
+# step 2: copypaste the following command into a-Shell:
+#   curl -L https://github.com/9001/copyparty/raw/refs/heads/hovudstraum/contrib/setup-ashell.sh | sh
+#
+# step 3: launch copyparty with this command: cpp
+#
+# if you ever want to upgrade copyparty, just repeat step 2
+
+
+
+cd "$HOME/Documents"
+curl -Locopyparty https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py
+
+
+
+# create the config file? (cannot use heredoc because body too large)
+[ -e cpc ] || {
+echo '[global]' >cpc
+echo '  p: 80, 443, 3923  # enable http and https on these ports' >>cpc
+echo '  e2dsa      # enable file indexing and filesystem scanning' >>cpc
+echo '  e2ts       # and enable multimedia indexing' >>cpc
+echo '  ver        # show copyparty version in the controlpanel' >>cpc
+echo '  qrz: 2     # enable qr-code and make it big' >>cpc
+echo '  qrp: 1     # reduce qr-code padding' >>cpc
+echo '  qr-fg: -1  # optimize for basic/simple terminals' >>cpc
+echo '  qr-wait: 0.3  # less chance of getting scrolled away' >>cpc
+echo '' >>cpc
+echo '  # enable these by uncommenting them:' >>cpc
+echo '  # ftp: 21    # enable ftp server on port 21' >>cpc
+echo '  # tftp: 69   # enable tftp server on port 69' >>cpc
+echo '' >>cpc
+echo '[/]' >>cpc
+echo '  ~/Documents' >>cpc
+echo '  accs:' >>cpc
+echo '    A: *' >>cpc
+}
+
+
+
+# create the launcher?
+[ -e cpp ] || {
+echo '#!/bin/sh' >cpp
+echo '' >>cpp
+echo '# change the font so the qr-code draws correctly:' >>cpp
+echo 'config -n "Menlo"  # name' >>cpp
+echo 'config -s 8  # size' >>cpp
+echo '' >>cpp
+echo '# launch copyparty' >>cpp
+echo 'exec copyparty -c cpc "$@"' >>cpp
+}
+
+
+
+chmod 755 copyparty cpp
+echo
+echo =================================
+echo
+echo 'okay, all done!'
+echo
+echo 'you can edit your config'
+echo 'with this command: vim cpc'
+echo
+echo 'you can run copyparty'
+echo 'with this command: cpp'
+echo

contrib/systemd/copyparty-user.service

@@ -1,5 +1,5 @@
 # this will start `/usr/bin/copyparty`
-# and read config from `$HOME/.config/copyparty.conf`
+# and read config from `$HOME/.config/copyparty/copyparty.conf`
 #
 # unless you add -q to disable logging, you may want to remove the
 #   following line to allow buffering (slightly better performance):
@@ -11,13 +11,13 @@ Description=copyparty file server
 [Service]
 Type=notify
 SyslogIdentifier=copyparty
-WorkingDirectory=/var/lib/copyparty-jail
+WorkingDirectory=/var/lib/copyparty-jail/srv
 Environment=PYTHONUNBUFFERED=x
 Environment=PRTY_CONFIG=%h/.config/copyparty/copyparty.conf
 ExecReload=/bin/kill -s USR1 $MAINPID
 
 # ensure there is a config
-ExecStartPre=/bin/bash -c 'if [[ ! -f %h/.config/copyparty/copyparty.conf ]]; then mkdir -p %h/.config/copyparty; cp /etc/copyparty/copyparty.conf %h/.config/copyparty/copyparty.conf; fi'
+ExecStartPre=/bin/bash -c 'if [[ ! -f %h/.config/copyparty/copyparty.conf ]]; then mkdir -p %h/.config/copyparty; cp -v /etc/copyparty.conf %h/.config/copyparty/; fi'
 
 # run copyparty
 ExecStart=/usr/bin/python3 /usr/bin/copyparty

contrib/systemd/copyparty.conf

@@ -1,13 +1,15 @@
-[global]
-  i: 127.0.0.1
-
-[accounts]
-  user: password
-
 [/]
-  /var/lib/copyparty-jail
+  /var/lib/copyparty-jail/srv/
   accs:
-    r: *
-    rwdma: user
-  flags:
-    grid
+    r: *    # everyone can [r]ead  (browse/download)
+
+
+# above is a very simple config;
+# the webroot is /var/lib/copyparty-jail/srv/
+# readable by everyone, no login required
+#
+# you'll want to replace this file; some inspiration:
+# - a basic example config: https://github.com/9001/copyparty/blob/hovudstraum/contrib/systemd/copyparty.example.conf
+# - another example with focus on syntax: https://github.com/9001/copyparty/blob/hovudstraum/docs/example.conf
+# - and CTRL-F "config file example" here: https://github.com/9001/copyparty
+# - full list of options: https://copyparty.eu/cli/

contrib/systemd/copyparty.example.conf

@@ -3,9 +3,6 @@
 # vim: ft=yaml:
 
 
-# put this file in /etc/
-
-
 [global]
   e2dsa  # enable file indexing and filesystem scanning
   e2ts   # and enable multimedia indexing
@@ -16,7 +13,16 @@
   # and copyparty replaces %Y-%m%d with Year-MonthDay, so the
   # full path will be something like /var/log/copyparty/2023-1130.txt
   # (note: enable compression by adding .xz at the end)
-  q, lo: $LOGS_DIRECTORY/%Y-%m%d.log
+  q, lo: ${LOGS_DIRECTORY}/%Y-%m%d.log
+
+  # enable version-checker by uncommenting one of the 'vc-url' lines below; this will
+  # periodically check if your copyparty version has a known security vulnerability,
+  # showing a warning on /?h (control-panel) for all users with permission 'a' or 'A'
+  #vc-url: https://api.github.com/repos/9001/copyparty/security-advisories?per_page=9
+  #vc-url: https://api.copyparty.eu/advisories
+
+  vc-age: 3  # how many hours to wait between each version-check
+  vc-exit  # emergency-exit if a version-check indicates that the current version is vulnerable
 
   # p: 80,443,3923   # listen on 80/443 as well (requires CAP_NET_BIND_SERVICE)
   # i: 127.0.0.1     # only allow connections from localhost (reverse-proxies)
@@ -40,3 +46,5 @@
   accs:
     rw: *      # everyone gets read-write access, but
     rwmda: ed  # the user "ed" gets read-write-move-delete-admin
+  flags:
+    e2ds       # enable filesystem-scanning for this volume only

contrib/systemd/copyparty@.service

@@ -1,5 +1,5 @@
 # this will start `/usr/bin/copyparty`
-# and read config from `/etc/copyparty/copyparty.conf`
+# and read config from `/etc/copyparty.conf`
 #
 # the %i refers to whatever you put after the copyparty@
 #   so with copyparty@foo.service, %i == foo
@@ -14,9 +14,9 @@ Description=copyparty file server
 [Service]
 Type=notify
 SyslogIdentifier=copyparty
-WorkingDirectory=/var/lib/copyparty-jail
+WorkingDirectory=/var/lib/copyparty-jail/srv
 Environment=PYTHONUNBUFFERED=x
-Environment=PRTY_CONFIG=/etc/copyparty/copyparty.conf
+Environment=PRTY_CONFIG=/etc/copyparty.conf
 ExecReload=/bin/kill -s USR1 $MAINPID
 
 # user to run as + where the TLS certificate is (if any)

contrib/systemd/index.md

@@ -1,10 +1,13 @@
-this is `/var/lib/copyparty-jail`, the fallback webroot when copyparty has not yet been configured
+this is `/var/lib/copyparty-jail/srv/`, the fallback webroot when copyparty has not yet been configured
 
-please edit `/etc/copyparty/copyparty.conf` (if running as a system service)
-or `$HOME/.config/copyparty/copyparty.conf` if running as a user service
+please edit the config-file and restart copyparty;
+* if running as a system service: `vim /etc/copyparty.conf`
+* if running as a user service: `vim $HOME/.config/copyparty/copyparty.conf`
 
-a basic configuration example is available at https://github.com/9001/copyparty/blob/hovudstraum/contrib/systemd/copyparty.example.conf
-a configuration example that explains most flags is available at https://github.com/9001/copyparty/blob/hovudstraum/docs/chungus.conf
+some inspiration:
+* a basic example config: https://github.com/9001/copyparty/blob/hovudstraum/contrib/systemd/copyparty.example.conf
+* another example with focus on syntax: https://github.com/9001/copyparty/blob/hovudstraum/docs/example.conf
+* and CTRL-F `config file example` here: https://github.com/9001/copyparty
 
-the full list of configuration options can be seen at https://ocv.me/copyparty/helptext.html 
+the full list of configuration options can be seen at https://copyparty.eu/cli/ 
 or by running `copyparty --help`

contrib/systemd/prisonparty@.service

@@ -1,6 +1,6 @@
 # this will start `/usr/bin/copyparty`
 # in a chroot, preventing accidental access elsewhere,
-# and read copyparty config from `/etc/copyparty/copyparty.conf`
+# and read copyparty config from `/etc/copyparty.conf`
 #
 # expose additional filesystem locations to copyparty
 #   by listing them between the last `%i` and `--`
@@ -19,9 +19,9 @@ Description=copyparty file server
 [Service]
 Type=notify
 SyslogIdentifier=prisonparty
-WorkingDirectory=/var/lib/copyparty-jail
+WorkingDirectory=/var/lib/copyparty-jail/srv
 Environment=PYTHONUNBUFFERED=x
-Environment=PRTY_CONFIG=/etc/copyparty/copyparty.conf
+Environment=PRTY_CONFIG=/etc/copyparty.conf
 ExecReload=/bin/kill -s USR1 $MAINPID
 
 # user to run as + where the TLS certificate is (if any)

contrib/traefik/traefik-docker/copyparty.compose.yml

@@ -0,0 +1,36 @@
+# docker-compose.yml for copyparty
+services:
+  copyparty:
+    image: copyparty/ac:latest
+    container_name: copyparty
+    user: "1000:1000"
+    volumes:
+      - ./:/cfg:z
+      - /path/to/your/fileshare/top/folder:/w:z
+  
+    environment:
+      LD_PRELOAD: /usr/lib/libmimalloc-secure.so.NOPE
+      # enable mimalloc by replacing "NOPE" with "2" for a nice speed-boost (will use twice as much ram)
+
+      PYTHONUNBUFFERED: 1
+      # ensures log-messages are not delayed (but can reduce speed a tiny bit)
+
+    stop_grace_period: 15s  # thumbnailer is allowed to continue finishing up for 10s after the shutdown signal
+    healthcheck:
+      # hide it from logs with "/._" so it matches the default --lf-url filter 
+      test: ["CMD-SHELL", "wget --spider -q 127.0.0.1:3923/?reset=/._"]
+      interval: 1m
+      timeout: 2s
+      retries: 5
+      start_period: 15s
+    networks:
+      - traefik_default
+    labels:
+      - traefik.http.routers.copyparty.rule=Host(`<DOMAIN>`)
+      - traefik.http.services.copyparty.loadbalancer.server.port=3923
+      - traefik.http.routers.copyparty.tls.certresolver=le
+      - traefik.http.routers.copyparty.entrypoints=websecure
+  
+networks:
+  traefik_default:
+    external: true

contrib/traefik/traefik-docker/copyparty.conf

@@ -0,0 +1,43 @@
+# note: copied from copyparty/docs/examples/docker/basic-docker-compose/copyparty.conf
+# not actually YAML but lets pretend:
+# -*- mode: yaml -*-
+# vim: ft=yaml:
+
+
+[global]
+  e2dsa  # enable file indexing and filesystem scanning
+  e2ts   # enable multimedia indexing
+  ansi   # enable colors in log messages (both in logfiles and stdout)
+
+  # q, lo: /cfg/log/%Y-%m%d.log   # log to file instead of docker
+
+  # p: 3939          # listen on another port
+  # ipa: 10.89.      # only allow connections from 10.89.*
+  # ipa: 172.16.4.0/23  # ...or only 172.16.4.* and 172.16.5.*
+  # ipa: lan         # ...or allow LAN only; reject internet IPs
+  # df: 16           # stop accepting uploads if less than 16 GB free disk space
+  # ver              # show copyparty version in the controlpanel
+  # grid             # show thumbnails/grid-view by default
+  # theme: 2         # monokai
+  # name: datasaver  # change the server-name that's displayed in the browser
+  # stats, nos-dup   # enable the prometheus endpoint, but disable the dupes counter (too slow)
+  # no-robots, force-js  # make it harder for search engines to read your server
+
+  # enable version-checking by uncommenting one of the vc-url lines below;
+  # shows a warning-banner in the controlpanel if your version has a known vulnerability
+  #vc-url: https://api.github.com/repos/9001/copyparty/security-advisories?per_page=9
+  #vc-url: https://api.copyparty.eu/advisories
+  vc-exit  # panic and shutdown instead of just showing the warning
+
+
+[accounts]
+  ed: wark  # username: password
+
+
+[/]            # create a volume at "/" (the webroot), which will
+  /w           # share /w (the docker data volume)
+  accs:
+    rw: *      # everyone gets read-write access, but
+    rwmda: ed  # the user "ed" gets read-write-move-delete-admin
+  flags:
+    e2ds       # enable filesystem-scanning for this volume only

contrib/traefik/traefik-docker/traefik.compose.yml

@@ -0,0 +1,34 @@
+# docker-compose.yml for traefik
+services:
+  traefik:
+    image: traefik:latest
+    command:
+      - --api.insecure=false
+      - --api.dashboard=true
+      - --providers.docker=true
+      - --entrypoints.web.address=:80
+      - --entrypoints.websecure.address=:443
+      - --entryPoints.websecure.http.tls=true
+      - --entryPoints.web.http.redirections.entryPoint.to=websecure
+      - --entryPoints.web.http.redirections.entryPoint.scheme=https
+      - --certificatesresolvers.le.acme.email=<your email here>
+      - --certificatesresolvers.le.acme.storage=/letsencrypt/acme.json
+      - --certificatesresolvers.le.acme.httpchallenge.entrypoint=web
+    ports:
+      - 80:80
+      - 443:443
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ./letsencrypt:/letsencrypt
+    labels:
+      # OPTIONAL: host your traefik dashboard on <DOMAIN>
+      # - traefik.http.routers.traefik-dash.rule=Host(`<DOMAIN>`)
+      # - traefik.http.routers.traefik-dash.entrypoints=websecure
+      # - traefik.http.routers.traefik-dash.service=api@internal
+      # - traefik.http.routers.traefik-dash.tls.certresolver=le
+      # - traefik.http.routers.traefik-dash.middlewares=traefik-dash-auth,traefik-dash-redirect
+      # - traefik.http.middlewares.traefik-dash-auth.basicauth.users=<USERNAME>:<HTPASSWD_ENCODED_PASSWD>
+      # - traefik.http.middlewares.traefik-dash-redirect.redirectregex.regex=^https://<SUBDOMAIN>\.<DOMAIN>\.TLD/?$
+      # - traefik.http.middlewares.traefik-dash-redirect.redirectregex.replacement=https://<DOMAIN>/dashboard/
+
+networks: {}

copyparty/__init__.py

@@ -1,5 +1,5 @@
 # coding: utf-8
-from __future__ import print_function, unicode_literals
+from __future__ import division, print_function, unicode_literals
 
 import os
 import platform
@@ -40,10 +40,23 @@ VT100 = "--ansi" in sys.argv or (
 )
 # introduced in anniversary update
 
+zs = os.environ.get("PRTY_CONFIG", "")
+CFG_DEF = [zs] if zs else []
+
 ANYWIN = WINDOWS or sys.platform in ["msys", "cygwin"]
 
 MACOS = platform.system() == "Darwin"
 
+FREEBSD = platform.system() == "FreeBSD"
+
+OPENBSD = platform.system() == "OpenBSD"
+
+ANYBSD = FREEBSD or OPENBSD
+
+UNIX = MACOS or ANYBSD
+
+GRAAL = platform.python_implementation() == "GraalVM"
+
 EXE = bool(getattr(sys, "frozen", False))
 
 try:
@@ -55,7 +68,7 @@ except:
 zs = """
 web/a/partyfuse.py
 web/a/u2c.py
-web/a/webdav-cfg.bat
+web/a/webdav-cfg.txt
 web/baguettebox.js
 web/browser.css
 web/browser.html
@@ -67,9 +80,10 @@ web/deps/busy.mp3
 web/deps/easymde.css
 web/deps/easymde.js
 web/deps/marked.js
-web/deps/fuse.py
+web/deps/mfusepy.py
 web/deps/mini-fa.css
 web/deps/mini-fa.woff
+web/deps/orbitron.woff2
 web/deps/prism.css
 web/deps/prism.js
 web/deps/prismd.css
@@ -86,8 +100,8 @@ web/md2.js
 web/mde.css
 web/mde.html
 web/mde.js
-web/msg.css
 web/msg.html
+web/opds.xml
 web/rups.css
 web/rups.html
 web/rups.js
@@ -99,19 +113,48 @@ web/splash.html
 web/splash.js
 web/svcs.html
 web/svcs.js
+web/tl/chi.js
+web/tl/cze.js
+web/tl/deu.js
+web/tl/epo.js
+web/tl/fin.js
+web/tl/fra.js
+web/tl/grc.js
+web/tl/hun.js
+web/tl/ita.js
+web/tl/jpn.js
+web/tl/kor.js
+web/tl/nld.js
+web/tl/nno.js
+web/tl/nor.js
+web/tl/pol.js
+web/tl/por.js
+web/tl/rus.js
+web/tl/spa.js
+web/tl/swe.js
+web/tl/tur.js
+web/tl/ukr.js
+web/tl/vie.js
 web/ui.css
 web/up2k.js
 web/util.js
-web/w.hash.js
+web/wopi.html
 """
 RES = set(zs.strip().split("\n"))
+RESM = {
+    "web/a/partyfuse.txt": "web/a/partyfuse.py",
+    "web/a/u2c.txt": "web/a/u2c.py",
+    "web/a/webdav-cfg.bat": "web/a/webdav-cfg.txt",
+}
 
 
 class EnvParams(object):
     def __init__(self) -> None:
         self.t0 = time.time()
         self.mod = ""
+        self.mod_ = ""
         self.cfg = ""
+        self.scfg = True
 
 
 E = EnvParams()

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (1, 19, 2)
-CODENAME = "usernames"
-BUILD_DT = (2025, 8, 17)
+VERSION = (1, 20, 26)
+CODENAME = "sftp is fine too"
+BUILD_DT = (2026, 10, 14)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/bos/bos.py

@@ -1,19 +1,23 @@
 # coding: utf-8
-from __future__ import print_function, unicode_literals
+from __future__ import division, print_function, unicode_literals
 
 import os
+import time
 
 from ..util import SYMTIME, fsdec, fsenc
 from . import path as path
 
 if True:  # pylint: disable=using-constant-test
-    from typing import Any, Optional
+    from typing import Any, Optional, Union
+
+    from ..util import NamedLogger
 
 MKD_755 = {"chmod_d": 0o755}
 MKD_700 = {"chmod_d": 0o700}
+UTIME_CLAMPS = ((max, -2147483647), (max, 1), (min, 4294967294), (min, 2147483646))
 
-_ = (path, MKD_755, MKD_700)
-__all__ = ["path", "MKD_755", "MKD_700"]
+_ = (path, MKD_755, MKD_700, UTIME_CLAMPS)
+__all__ = ["path", "MKD_755", "MKD_700", "UTIME_CLAMPS"]
 
 # grep -hRiE '(^|[^a-zA-Z_\.-])os\.' . | gsed -r 's/ /\n/g;s/\(/(\n/g' | grep -hRiE '(^|[^a-zA-Z_\.-])os\.' | sort | uniq -c
 # printf 'os\.(%s)' "$(grep ^def bos/__init__.py | gsed -r 's/^def //;s/\(.*//' | tr '\n' '|' | gsed -r 's/.$//')"
@@ -36,7 +40,7 @@ def makedirs(name: str, vf: dict[str, Any] = MKD_755, exist_ok: bool = True) ->
     todo = []
     bname = fsenc(name)
     while bname:
-        if os.path.isdir(bname):
+        if os.path.isdir(bname) or bname in todo:
             break
         todo.append(bname)
         bname = os.path.dirname(bname)
@@ -99,6 +103,44 @@ def utime(
         return os.utime(fsenc(p), times)
 
 
+def utime_c(
+    log: Union["NamedLogger", Any],
+    p: str,
+    ts: float,
+    follow_symlinks: bool = True,
+    throw: bool = False,
+) -> Optional[float]:
+    clamp = 0
+    ov = ts
+    bp = fsenc(p)
+    now = time.time()
+    while True:
+        try:
+            if SYMTIME:
+                os.utime(bp, (now, ts), follow_symlinks=follow_symlinks)
+            else:
+                os.utime(bp, (now, ts))
+            if clamp:
+                t = "filesystem rejected utime(%r); clamped %s to %s"
+                log(t % (p, ov, ts))
+            return ts
+        except Exception as ex:
+            pv = ts
+            while clamp < len(UTIME_CLAMPS):
+                fun, cv = UTIME_CLAMPS[clamp]
+                ts = fun(ts, cv)
+                clamp += 1
+                if ts != pv:
+                    break
+            if clamp >= len(UTIME_CLAMPS):
+                if throw:
+                    raise
+                else:
+                    t = "could not utime(%r) to %s; %s, %r"
+                    log(t % (p, ov, ex, ex))
+                    return None
+
+
 if hasattr(os, "lstat"):
 
     def lstat(p: str) -> os.stat_result:

copyparty/bos/path.py

@@ -1,5 +1,5 @@
 # coding: utf-8
-from __future__ import print_function, unicode_literals
+from __future__ import division, print_function, unicode_literals
 
 import os
 

copyparty/broker_mp.py

@@ -1,5 +1,5 @@
 # coding: utf-8
-from __future__ import print_function, unicode_literals
+from __future__ import division, print_function, unicode_literals
 
 import threading
 import time
@@ -146,13 +146,13 @@ class BrokerMp(object):
         returns a Queue object which eventually contains the response if want_retval
         (not-impl here since nothing uses it yet)
         """
-        if dest == "httpsrv.listen":
-            for p in self.procs:
-                p.q_pend.put((0, dest, [args[0], len(self.procs)]))
-
-        elif dest == "httpsrv.set_netdevs":
-            for p in self.procs:
-                p.q_pend.put((0, dest, list(args)))
+        if dest.startswith("httpsrv."):
+            if dest == "httpsrv.listen":
+                for p in self.procs:
+                    p.q_pend.put((0, dest, [args[0], len(self.procs)]))
+            else:
+                for p in self.procs:
+                    p.q_pend.put((0, dest, list(args)))
 
         elif dest == "cb_httpsrv_up":
             self.hub.cb_httpsrv_up()
@@ -160,6 +160,13 @@ class BrokerMp(object):
         else:
             raise Exception("what is " + str(dest))
 
+    def say1(self, dest: str, *args: Any) -> None:
+        """
+        send message to one lucky recipient
+        """
+        p = self.procs[0]
+        p.q_pend.put((0, dest, list(args)))
+
     def periodic(self) -> None:
         while True:
             time.sleep(1)

copyparty/broker_mpw.py

@@ -1,5 +1,5 @@
 # coding: utf-8
-from __future__ import print_function, unicode_literals
+from __future__ import division, print_function, unicode_literals
 
 import argparse
 import os
@@ -12,6 +12,7 @@ import queue
 from .__init__ import ANYWIN
 from .authsrv import AuthSrv
 from .broker_util import BrokerCli, ExceptionalQueue, NotExQueue
+from .fsutil import ramdisk_chk
 from .httpsrv import HttpSrv
 from .util import FAKE_MP, Daemon, HMaccas
 
@@ -56,6 +57,7 @@ class MpWorker(BrokerCli):
 
         # starting to look like a good idea
         self.asrv = AuthSrv(args, None, False)
+        ramdisk_chk(self.asrv)
 
         # instantiate all services here (TODO: inheritance?)
         self.iphash = HMaccas(os.path.join(self.args.E.cfg, "iphash"), 8)
@@ -99,6 +101,7 @@ class MpWorker(BrokerCli):
             if dest == "reload":
                 self.logw("mpw.asrv reloading")
                 self.asrv.reload()
+                ramdisk_chk(self.asrv)
                 self.logw("mpw.asrv reloaded")
                 continue
 

copyparty/broker_thr.py

@@ -1,5 +1,5 @@
 # coding: utf-8
-from __future__ import print_function, unicode_literals
+from __future__ import division, print_function, unicode_literals
 
 import os
 import threading
@@ -53,12 +53,12 @@ class BrokerThr(BrokerCli):
         return NotExQueue(obj(*args))  # type: ignore
 
     def say(self, dest: str, *args: Any) -> None:
-        if dest == "httpsrv.listen":
-            self.httpsrv.listen(args[0], 1)
-            return
+        if dest.startswith("httpsrv."):
+            if dest == "httpsrv.listen":
+                self.httpsrv.listen(args[0], 1)
+                return
 
-        if dest == "httpsrv.set_netdevs":
-            self.httpsrv.set_netdevs(args[0])
+            getattr(self.httpsrv, dest[8:])(*args)
             return
 
         # new ipc invoking managed service in hub

copyparty/broker_util.py

@@ -1,8 +1,7 @@
 # coding: utf-8
-from __future__ import print_function, unicode_literals
+from __future__ import division, print_function, unicode_literals
 
 import argparse
-import traceback
 
 from queue import Queue
 
@@ -17,6 +16,7 @@ if True:  # pylint: disable=using-constant-test
 
 if TYPE_CHECKING:
     from .httpsrv import HttpSrv
+    from .svchub import SvcHub
 
 
 class ExceptionalQueue(Queue, object):
@@ -51,6 +51,7 @@ class BrokerCli(object):
     for example resolving httpconn.* in httpcli -- see lines tagged #mypy404
     """
 
+    hub: "SvcHub"
     log: "RootLogger"
     args: argparse.Namespace
     asrv: AuthSrv

copyparty/cert.py

@@ -1,12 +1,17 @@
+# coding: utf-8
+from __future__ import division, print_function, unicode_literals
+
 import calendar
 import errno
 import json
 import os
+import shutil
 import time
 
 from .__init__ import ANYWIN
 from .util import Netdev, atomic_move, load_resource, runcmd, wunlink
 
+NO_TLS = bool(os.environ.get("PRTY_NO_TLS"))
 HAVE_CFSSL = not os.environ.get("PRTY_NO_CFSSL")
 
 if True:  # pylint: disable=using-constant-test
@@ -19,6 +24,19 @@ else:
     VF = {"mv_re_t": 0, "rm_re_t": 0}
 
 
+def _sp_err(exe, what, rc, so, se, sin):
+    try:
+        zs = shutil.which(exe)
+    except:
+        zs = "<?>"
+    try:
+        zi = os.path.getsize(zs)
+    except:
+        zi = 0
+    t = "failed to %s; error %s using %s (%s):\n  STDOUT: %s\n  STDERR: %s\n  STDIN: %s\n"
+    raise Exception(t % (what, rc, zs, zi, so, se, sin.decode("utf-8")))
+
+
 def ensure_cert(log: "RootLogger", args) -> None:
     """
     the default cert (and the entire TLS support) is only here to enable the
@@ -37,9 +55,12 @@ def ensure_cert(log: "RootLogger", args) -> None:
         with open(args.cert, "wb") as f:
             f.write(cert_insec)
 
+    if args.certkey and not os.path.isfile(args.certkey):
+        raise Exception("certificate-key file does not exist: " + args.certkey)
+
     with open(args.cert, "rb") as f:
         buf = f.read()
-        o1 = buf.find(b" PRIVATE KEY-")
+        o1 = buf.find(b" PRIVATE KEY-") if not args.certkey else 0
         o2 = buf.find(b" CERTIFICATE-")
         m = "unsupported certificate format: "
         if o1 < 0:
@@ -53,8 +74,9 @@ def ensure_cert(log: "RootLogger", args) -> None:
         with open(args.cert, "rb") as f:
             active_cert = f.read()
         if active_cert == cert_insec:
-            t = "using default TLS certificate; https will be insecure:\033[36m {}"
-            log("cert", t.format(args.cert), 3)
+            t = "using default TLS certificate; %s will be insecure:\033[36m %s"
+            t2 = "https" if not args.ftps else "ftps"
+            log("cert", t % (t2, args.cert), 3)
     except:
         pass
 
@@ -107,13 +129,13 @@ def _gen_ca(log: "RootLogger", args):
     cmd = "cfssl gencert -initca -"
     rc, so, se = runcmd(cmd.split(), 30, sin=sin)
     if rc:
-        raise Exception("failed to create ca-cert: {}, {}".format(rc, se), 3)
+        _sp_err("cfssl", "create ca-cert", rc, so, se, sin)
 
     cmd = "cfssljson -bare ca"
     sin = so.encode("utf-8")
     rc, so, se = runcmd(cmd.split(), 10, sin=sin, cwd=args.crt_dir)
     if rc:
-        raise Exception("failed to translate ca-cert: {}, {}".format(rc, se), 3)
+        _sp_err("cfssljson", "translate ca-cert", rc, so, se, sin)
 
     bname = os.path.join(args.crt_dir, "ca")
     try:
@@ -130,6 +152,7 @@ def _gen_srv(log: "RootLogger", args, netdevs: dict[str, Netdev]):
     nlog: "NamedLogger" = lambda msg, c=0: log("cert-gen-srv", msg, c)
 
     names = args.crt_ns.split(",") if args.crt_ns else []
+    names = [x.strip() for x in names]
     if not args.crt_exact:
         for n in names[:]:
             names.append("*.{}".format(n))
@@ -200,13 +223,13 @@ def _gen_srv(log: "RootLogger", args, netdevs: dict[str, Netdev]):
     acmd = cmd.split() + ["-hostname=" + ",".join(names), "-"]
     rc, so, se = runcmd(acmd, 30, sin=sin, cwd=args.crt_dir)
     if rc:
-        raise Exception("failed to create cert: {}, {}".format(rc, se))
+        _sp_err("cfssl", "create cert", rc, so, se, sin)
 
     cmd = "cfssljson -bare srv"
     sin = so.encode("utf-8")
     rc, so, se = runcmd(cmd.split(), 10, sin=sin, cwd=args.crt_dir)
     if rc:
-        raise Exception("failed to translate cert: {}, {}".format(rc, se))
+        _sp_err("cfssljson", "translate cert", rc, so, se, sin)
 
     bname = os.path.join(args.crt_dir, "srv")
     try:
@@ -234,10 +257,10 @@ def _gen_srv(log: "RootLogger", args, netdevs: dict[str, Netdev]):
 def gencert(log: "RootLogger", args, netdevs: dict[str, Netdev]):
     global HAVE_CFSSL
 
-    if args.http_only:
+    if args.http_only and not args.ftps:
         return
 
-    if args.no_crt or not HAVE_CFSSL:
+    if args.no_crt or args.certkey or not HAVE_CFSSL:
         ensure_cert(log, args)
         return
 

copyparty/cfg.py

@@ -1,5 +1,5 @@
 # coding: utf-8
-from __future__ import print_function, unicode_literals
+from __future__ import division, print_function, unicode_literals
 
 # awk -F\" '/add_argument\("-[^-]/{print(substr($2,2))}' copyparty/__main__.py | sort | tr '\n' ' '
 zs = "a c e2d e2ds e2dsa e2t e2ts e2tsr e2v e2vp e2vu ed emp i j lo mcr mte mth mtm mtp nb nc nid nih nth nw p q s ss sss v z zv"
@@ -19,16 +19,25 @@ def vf_bmap() -> dict[str, str]:
         "no_clone": "noclone",
         "no_dirsz": "nodirsz",
         "no_dupe": "nodupe",
+        "no_dupe_m": "nodupem",
         "no_forget": "noforget",
+        "no_html": "nohtml",
+        "no_mime": "nomime",
         "no_pipe": "nopipe",
         "no_robots": "norobots",
+        "no_script": "noscript",
         "no_tail": "notail",
         "no_thumb": "dthumb",
         "no_vthumb": "dvthumb",
         "no_athumb": "dathumb",
+        "vol_nospawn": "nospawn",
+        "vol_or_crash": "assert_root",
     }
     for k in (
+        "allow_svg_js",
         "dedup",
+        "dlni",
+        "dothidden",
         "dotsrch",
         "e2d",
         "e2ds",
@@ -43,19 +52,37 @@ def vf_bmap() -> dict[str, str]:
         "grid",
         "gsel",
         "hardlink",
+        "hist_cow",
         "magic",
+        "md_no_br",
         "no_db_ip",
+        "no_logues",
+        "no_readme",
         "no_sb_md",
         "no_sb_lg",
         "nsort",
         "og",
         "og_no_head",
         "og_s_title",
+        "opds",
         "rand",
+        "redup_dry",
         "reflink",
+        "rm_partial",
         "rmagic",
         "rss",
+        "show_hist",
+        "ui_noacci",
+        "ui_nocpla",
+        "ui_nolbar",
+        "ui_nombar",
+        "ui_nonav",
+        "ui_notree",
+        "ui_norepl",
+        "ui_nosrvi",
+        "ui_noctxb",
         "wo_up_readme",
+        "wram",
         "xdev",
         "xlink",
         "xvol",
@@ -79,18 +106,30 @@ def vf_vmap() -> dict[str, str]:
         "th_x3": "th3x",
     }
     for k in (
+        "apnd_who",
         "bup_ck",
+        "cachectl",
+        "casechk",
         "chmod_d",
         "chmod_f",
+        "csp_ui",
+        "csp_dl",
         "dbd",
+        "db_xattr",
+        "du_who",
+        "epilogues",
+        "ufavico",
         "forget_ip",
+        "fsnt",
         "hsortn",
         "html_head",
+        "html_head_s",
         "lg_sbf",
         "md_sbf",
         "lg_sba",
         "md_sba",
         "md_hist",
+        "md_nhist",
         "nrand",
         "u2ow",
         "og_desc",
@@ -102,16 +141,31 @@ def vf_vmap() -> dict[str, str]:
         "og_title_i",
         "og_tpl",
         "og_ua",
+        "opds_exts",
+        "prologues",
+        "preadmes",
         "put_ck",
         "put_name",
+        "readmes",
+        "redup",
+        "rotf_tz",
         "mv_retry",
         "rm_retry",
+        "rss_sort",
+        "rss_fmt_t",
+        "rss_fmt_d",
+        "rw_edit",
+        "shr_who",
         "sort",
         "tail_fd",
         "tail_rate",
         "tail_tmax",
         "tail_who",
         "tcolor",
+        "th_covers",
+        "th_pregen",
+        "th_qv",
+        "th_qvx",
         "th_spec_p",
         "txt_eol",
         "unlist",
@@ -177,14 +231,21 @@ flagcats = {
         "hardlinkonly": "dedup with hardlink only, never symlink;\nmake a full copy if hardlink is impossible",
         "reflink": "enable reflink-based file deduplication,\nwith fallback on full copy when that is impossible",
         "safededup": "verify on-disk data before using it for dedup",
+        "redup=A,B=T": "convert dedup-type for existing files",
+        "redup_dry": "dry-run mode for redup",
         "noclone": "take dupe data from clients, even if available on HDD",
         "nodupe": "rejects existing files (instead of linking/cloning them)",
+        "nodupem": "rejects existing files during moves as well",
+        "casechk=auto": "actively prevent case-insensitive filesystem? y/n",
         "chmod_d=755": "unix-permission for new dirs/folders",
         "chmod_f=644": "unix-permission for new files",
         "uid=573": "change owner of new files/folders to unix-user 573",
         "gid=999": "change owner of new files/folders to unix-group 999",
+        "fsnt=auto": "filesystem filename traits (lin/win/mac/auto)",
+        "wram": "allow uploading into ramdisks",
         "sparse": "force use of sparse files, mainly for s3-backed storage",
         "nosparse": "deny use of sparse files, mainly for slow storage",
+        "rm_partial": "delete unfinished uploads from HDD when they timeout",
         "daw": "enable full WebDAV write support (dangerous);\nPUT-operations will now \033[1;31mOVERWRITE\033[0;35m existing files",
         "nosub": "forces all uploads into the top folder of the vfs",
         "magic": "enables filetype detection for nameless uploads",
@@ -196,6 +257,7 @@ flagcats = {
         "pk": "forces server-side compression, optional arg: xz,9",
     },
     "upload rules": {
+        "apnd_who=dw": "who can append? (aw/dw/w/no)",
         "maxn=250,600": "max 250 uploads over 15min",
         "maxb=1g,300": "max 1 GiB over 5min (suffixes: b, k, m, g, t)",
         "vmaxb=1g": "total volume size max 1 GiB (suffixes: b, k, m, g, t)",
@@ -213,6 +275,7 @@ flagcats = {
     "upload rotation\n(moves all uploads into the specified folder structure)": {
         "rotn=100,3": "3 levels of subfolders with 100 entries in each",
         "rotf=%Y-%m/%d-%H": "date-formatted organizing",
+        "rotf_tz=Europe/Oslo": "timezone (default=UTC)",
         "lifetime=3600": "uploads are deleted after 1 hour",
     },
     "database, general": {
@@ -241,12 +304,14 @@ flagcats = {
         "no_db_ip": "never store uploader-IP in the db; disables unpost",
         "fat32": "avoid excessive reindexing on android sdcardfs",
         "dbd=[acid|swal|wal|yolo]": "database speed-durability tradeoff",
+        "hist_cow": "btrfs: keep copy-on-write enabled on db/thumbs",
         "xlink": "cross-volume dupe detection / linking (dangerous)",
         "xdev": "do not descend into other filesystems",
         "xvol": "do not follow symlinks leaving the volume root",
         "dotsrch": "show dotfiles in search results",
         "nodotsrch": "hide dotfiles in search results (default)",
         "srch_excl": "exclude search results with URL matching this regex",
+        "db_xattr=user.foo,user.bar": "index file xattrs as media-tags",
     },
     'database, audio tags\n"mte", "mth", "mtp", "mtm" all work the same as -mte, -mth, ...': {
         "mte=artist,title": "media-tags to index/display",
@@ -259,14 +324,19 @@ flagcats = {
         "dvthumb": "disables video thumbnails",
         "dathumb": "disables audio thumbnails (spectrograms)",
         "dithumb": "disables image thumbnails",
+        "dethumb": "disables custom thumbnails (--th-extract)",
         "pngquant": "compress audio waveforms 33% better",
         "thsize": "thumbnail res; WxH",
         "crop": "center-cropping (y/n/fy/fn)",
         "th3x": "3x resolution (y/n/fy/fn)",
+        "th_qv=40": "webp/jpg thumbnail quality (10~90)",
+        "th_qvx=40": "jxl thumbnail quality (10~90)",
+        "th_covers=a.jpg,a.png": "filenames to prefer as folder-thumbs",
         "convt": "convert-to-image timeout in seconds",
         "aconvt": "convert-to-audio timeout in seconds",
         "th_spec_p=1": "make spectrograms? 0=never 1=fallback 2=always",
         "ext_th=s=/b.png": "use /b.png as thumbnail for file-extension s",
+        "th_pregen=w,wf": "pregenerate thumbs for these formats",
     },
     "handlers\n(better explained in --help-handlers)": {
         "on404=PY": "handle 404s by executing PY file",
@@ -291,14 +361,27 @@ flagcats = {
         "sort": "default sort order",
         "nsort": "natural-sort of leading digits in filenames",
         "hsortn": "number of sort-rules to add to media URLs",
+        "ufavico=URL": "per-volume favicon (.ico/png/gif/svg)",
         "unlist": "dont list files matching REGEX",
+        "dothidden": "enable support for .hidden files",
+        "dlni": "force-download (no-inline) files on click",
         "html_head=TXT": "includes TXT in the <head>, or @PATH for file at PATH",
+        "html_head_s=TXT": "additional static text in the html <head>",
         "tcolor=#fc0": "theme color (a hint for webbrowsers, discord, etc.)",
         "nodirsz": "don't show total folder size",
+        "du_who=all": "show disk-usage info to everyone",
         "robots": "allows indexing by search engines (default)",
         "norobots": "kindly asks search engines to leave",
         "unlistcr": "don't list read-access in controlpanel",
         "unlistcw": "don't list write-access in controlpanel",
+        "prologues=.prologue.html": "files to embed above/before files",
+        "epilogues=.epilogue.html": "files to embed below/after files",
+        "readmes=readme.md,README.md": "files to embed as readmes",
+        "preadmes=preadme.md,PREADME.md": "files to embed as preadmes",
+        "plainreadme": "embed readme/preadme as monospace plaintext",
+        "plainlogues": "embed prologue/epilogue as sans-serif plaintext",
+        "no_readme": "disable embedding readme/preadme",
+        "no_logues": "disable embedding prologue/epilogue",
         "no_sb_md": "disable js sandbox for markdown files",
         "no_sb_lg": "disable js sandbox for prologue/epilogue",
         "sb_md": "enable js sandbox for markdown files (default)",
@@ -307,7 +390,21 @@ flagcats = {
         "lg_sbf": "list of *logue-sandbox safeguards to disable",
         "md_sba": "value of iframe allow-prop for markdown-sandbox",
         "lg_sba": "value of iframe allow-prop for *logue-sandbox",
+        "csp_ui": "content-security-policy for the web-UI",
+        "csp_dl": "content-security-policy for static files",
         "nohtml": "return html and markdown as text/html",
+        "nomime": "disallow ?mime to change response type",
+        "noscript": "disable most javascript by use of CSP",
+        "allow_svg_js": "allow javascript in svg images",
+        "ui_noacci": "hide account-info in the UI",
+        "ui_nocpla": "hide cpanel-link in the UI",
+        "ui_nolbar": "hide link-bar in the UI",
+        "ui_nombar": "hide top-menu in the UI",
+        "ui_nonav": "hide navpane+breadcrumbs in the UI",
+        "ui_notree": "hide navpane in the UI",
+        "ui_norepl": "hide repl-button in the UI",
+        "ui_nosrvi": "hide server-info in the UI",
+        "ui_noctxb": "hide context-buttons in the UI",
     },
     "opengraph (discord embeds)": {
         "og": "enable OG (disables hotlinking)",
@@ -323,8 +420,16 @@ flagcats = {
         "og_no_head": "you want to add tags manually with og_tpl",
         "og_ua": "if defined: only send OG html if useragent matches this regex",
     },
+    "opds": {
+        "opds": "enable OPDS",
+        "opds_exts": "file formats to list in OPDS feeds; leave empty to show everything",
+    },
     "textfiles": {
-        "md_hist": "where to put markdown backups; s=subfolder, v=volHist, n=nope",
+        "rw_edit=md,txt": "only require read+write to edit .md and .txt",
+        "md_no_br": "newline only on double-newline or two tailing spaces",
+        "md_hist": "where to put text-editor backups; s=subfolder, v=volHist, n=nope",
+        "md_nhist=99": "how many text-editor backups to keep",
+        "show_hist": "allow browsing the .hist folder (old textfiles)",
         "exp": "enable textfile expansion; see --help-exp",
         "exp_md": "placeholders to expand in markdown files; see --help",
         "exp_lg": "placeholders to expand in prologue/epilogue; see --help",
@@ -337,6 +442,12 @@ flagcats = {
         "tail_tmax=30": "kill connection after 30 sec",
         "tail_who=2": "restrict ?tail access (1=admins,2=authed,3=everyone)",
     },
+    "rss": {
+        "rss": "allow '?rss' URL suffix (experimental)",
+        "rss_sort=m": "default sort-order (m/u/n/s)",
+        "rss_fmt_t={fname}": "default title-format",
+        "rss_fmt_d={album},{.tn}": "default description-format",
+    },
     "others": {
         "dots": "allow all users with read-access to\nenable the option to show dotfiles in listings",
         "fk=8": 'generates per-file accesskeys,\nwhich are then required at the "g" permission;\nkeys are invalidated if filesize or inode changes',
@@ -344,8 +455,8 @@ flagcats = {
         "dk=8": 'generates per-directory accesskeys,\nwhich are then required at the "g" permission;\nkeys are invalidated if filesize or inode changes',
         "dks": "per-directory accesskeys allow browsing into subdirs",
         "dky": 'allow seeing files (not folders) inside a specific folder\nwith "g" perm, and does not require a valid dirkey to do so',
-        "rss": "allow '?rss' URL suffix (experimental)",
         "rmagic": "expensive analysis for mimetype accuracy",
+        "shr_who=auth": "who can create shares? no/auth/a",
         "unp_who=2": "unpost only if same... 1=ip+name, 2=ip, 3=name",
         "ups_who=2": "restrict viewing the list of recent uploads",
         "zip_who=2": "restrict access to download-as-zip/tar",
@@ -354,8 +465,11 @@ flagcats = {
         "zipmaxt=no": "reply with 'no' if download-as-zip exceeds max",
         "zipmaxu": "zip-size-limit does not apply to authenticated users",
         "nopipe": "disable race-the-beam (download unfinished uploads)",
+        "cachectl=no-cache": "controls caching in webbrowsers",
         "mv_retry": "ms-windows: timeout for renaming busy files",
         "rm_retry": "ms-windows: timeout for deleting busy files",
+        "nospawn": "don't create volume's folder if not exist",
+        "assert_root": "crash on startup if volume's folder not exist",
         "davauth": "ask webdav clients to login for all folders",
         "davrt": "show lastmod time of symlink destination, not the link itself\n(note: this option is always enabled for recursive listings)",
     },

copyparty/dxml.py

@@ -1,5 +1,5 @@
 # coding: utf-8
-from __future__ import print_function, unicode_literals
+from __future__ import division, print_function, unicode_literals
 
 import importlib
 import sys

copyparty/fsutil.py

@@ -1,15 +1,16 @@
 # coding: utf-8
-from __future__ import print_function, unicode_literals
+from __future__ import division, print_function, unicode_literals
 
 import argparse
+import json
 import os
 import re
 import time
 
-from .__init__ import ANYWIN, MACOS
-from .authsrv import AXS, VFS
+from .__init__ import ANYWIN, FREEBSD, MACOS, UNIX
+from .authsrv import AXS, VFS, AuthSrv
 from .bos import bos
-from .util import chkcmd, min_ex, undot
+from .util import chkcmd, fsenc, json_hesc, min_ex, runcmd, undot
 
 if True:  # pylint: disable=using-constant-test
     from typing import Optional, Union
@@ -17,23 +18,31 @@ if True:  # pylint: disable=using-constant-test
     from .util import RootLogger, undot
 
 
+PTN_FILEFRAG = re.compile(
+    r"^\s*[0-9]+:\s*[0-9]+\.\.\s*[0-9]+:\s*([0-9]+)\.\.\s*([0-9]+):\s*[0-9]+:\s*(.*)$"
+)
+
+
 class Fstab(object):
-    def __init__(self, log: "RootLogger", args: argparse.Namespace):
+    def __init__(self, log: "RootLogger", args: argparse.Namespace, verbose: bool):
         self.log_func = log
+        self.verbose = verbose
 
         self.warned = False
         self.trusted = False
         self.tab: Optional[VFS] = None
         self.oldtab: Optional[VFS] = None
         self.srctab = "a"
-        self.cache: dict[str, str] = {}
+        self.cache: dict[str, tuple[str, str]] = {}
         self.age = 0.0
         self.maxage = args.mtab_age
 
     def log(self, msg: str, c: Union[int, str] = 0) -> None:
+        if not c or self.verbose:
+            return
         self.log_func("fstab", msg, c)
 
-    def get(self, path: str) -> str:
+    def get(self, path: str) -> tuple[str, str]:
         now = time.time()
         if now - self.age > self.maxage or len(self.cache) > 9000:
             self.age = now
@@ -41,6 +50,7 @@ class Fstab(object):
             self.tab = None
             self.cache = {}
 
+        mp = ""
         fs = "ext4"
         msg = "failed to determine filesystem at %r; assuming %s\n%s"
 
@@ -50,7 +60,7 @@ class Fstab(object):
                 path = self._winpath(path)
             except:
                 self.log(msg % (path, fs, min_ex()), 3)
-                return fs
+                return fs, ""
 
         path = undot(path)
         try:
@@ -59,14 +69,14 @@ class Fstab(object):
             pass
 
         try:
-            fs = self.get_w32(path) if ANYWIN else self.get_unix(path)
+            fs, mp = self.get_w32(path) if ANYWIN else self.get_unix(path)
         except:
             self.log(msg % (path, fs, min_ex()), 3)
 
         fs = fs.lower()
-        self.cache[path] = fs
-        self.log("found %s at %r" % (fs, path))
-        return fs
+        self.cache[path] = (fs, mp)
+        self.log("found %s at %r, %r" % (fs, mp, path))
+        return fs, mp
 
     def _winpath(self, path: str) -> str:
         # try to combine volume-label + st_dev (vsn)
@@ -81,34 +91,67 @@ class Fstab(object):
         self.tab = VFS(self.log_func, "idk", "/", "/", AXS(), {})
         self.trusted = False
 
-    def build_tab(self) -> None:
-        self.log("inspecting mtab for changes")
-
+    def _from_sp_mount(self) -> dict[str, str]:
         sptn = r"^.*? on (.*) type ([^ ]+) \(.*"
-        if MACOS:
+        if MACOS or FREEBSD:
             sptn = r"^.*? on (.*) \(([^ ]+), .*"
 
         ptn = re.compile(sptn)
         so, _ = chkcmd(["mount"])
-        tab1: list[tuple[str, str]] = []
-        atab = []
+        dtab: dict[str, str] = {}
         for ln in so.split("\n"):
             m = ptn.match(ln)
             if not m:
                 continue
 
             zs1, zs2 = m.groups()
-            tab1.append((str(zs1), str(zs2)))
-            atab.append(ln)
+            dtab[str(zs1)] = str(zs2)
+
+        return dtab
+
+    def _from_proc(self) -> dict[str, str]:
+        ret: dict[str, str] = {}
+        with open("/proc/self/mounts", "rb", 262144) as f:
+            src = f.read(262144).decode("utf-8", "replace").split("\n")
+        for zsl in [x.split(" ") for x in src]:
+            if len(zsl) < 3:
+                continue
+            zs = zsl[1]
+            zs = zs.replace("\\011", "\t").replace("\\040", " ").replace("\\134", "\\")
+            ret[zs] = zsl[2]
+        return ret
+
+    def build_tab(self) -> None:
+        self.log("inspecting mtab for changes")
+        dtab = self._from_sp_mount() if UNIX else self._from_proc()
 
         # keep empirically-correct values if mounttab unchanged
-        srctab = "\n".join(sorted(atab))
+        srctab = str(sorted(dtab.items()))
         if srctab == self.srctab:
             self.tab = self.oldtab
             return
 
         self.log("mtab has changed; reevaluating support for sparse files")
 
+        try:
+            fuses = [mp for mp, fs in dtab.items() if fs == "fuseblk"]
+            if not fuses or UNIX:
+                raise Exception()
+            try:
+                so, _ = chkcmd(["lsblk", "-nrfo", "FSTYPE,MOUNTPOINT"])  # centos6
+            except:
+                so, _ = chkcmd(["lsblk", "-nrfo", "FSTYPE,MOUNTPOINTS"])  # future
+            for ln in so.split("\n"):
+                zsl = ln.split(" ", 1)
+                if len(zsl) != 2:
+                    continue
+                fs, mp = zsl
+                if mp in fuses:
+                    dtab[mp] = fs
+        except:
+            pass
+
+        tab1 = list(dtab.items())
         tab1.sort(key=lambda x: (len(x[0]), x[0]))
         path1, fs1 = tab1[0]
         tab = VFS(self.log_func, fs1, path1, path1, AXS(), {})
@@ -146,7 +189,7 @@ class Fstab(object):
             vn.realpath = ptn.sub(nval, vn.realpath)
             visit.extend(list(vn.nodes.values()))
 
-    def get_unix(self, path: str) -> str:
+    def get_unix(self, path: str) -> tuple[str, str]:
         if not self.tab:
             try:
                 self.build_tab()
@@ -155,20 +198,88 @@ class Fstab(object):
                 # prisonparty or other restrictive environment
                 if not self.warned:
                     self.warned = True
-                    self.log("failed to build tab:\n{}".format(min_ex()), 3)
+                    t = "failed to associate fs-mounts with the VFS (this is fine):\n%s"
+                    self.log(t % (min_ex(),), 6)
                 self.build_fallback()
 
         assert self.tab  # !rm
         ret = self.tab._find(path)[0]
         if self.trusted or path == ret.vpath:
-            return ret.realpath.split("/")[0]
+            return ret.realpath.split("/")[0], ret.vpath
         else:
-            return "idk"
+            return "idk", ""
 
-    def get_w32(self, path: str) -> str:
+    def get_w32(self, path: str) -> tuple[str, str]:
         if not self.tab:
             self.build_fallback()
 
         assert self.tab  # !rm
         ret = self.tab._find(path)[0]
-        return ret.realpath
+        return ret.realpath, ""
+
+
+_fstab: Optional[Fstab] = None
+winfs = set(("msdos", "vfat", "ntfs", "exfat"))
+# "msdos" = vfat on macos
+
+
+def ramdisk_chk(asrv: AuthSrv) -> None:
+    # should have been in authsrv but that's a circular import
+    global _fstab
+    mods = []
+    ramfs = ("tmpfs", "overlay")
+    log = asrv.log_func or print
+    if not _fstab:
+        _fstab = Fstab(log, asrv.args, False)
+    for vn in asrv.vfs.all_nodes.values():
+        if not vn.axs.uwrite or "wram" in vn.flags:
+            continue
+        ap = vn.realpath
+        if not ap or os.path.isfile(ap):
+            continue
+        fs, mp = _fstab.get(ap)
+        mp = "/" + mp.strip("/")
+        if fs == "tmpfs" or (mp == "/" and fs in ramfs):
+            mods.append((vn.vpath, ap, fs, mp))
+            vn.axs.uwrite.clear()
+            vn.axs.umove.clear()
+            for un, ztsp in list(vn.uaxs.items()):
+                zsl = list(ztsp)
+                zsl[1] = False
+                zsl[2] = False
+                vn.uaxs[un] = tuple(zsl)  # type: ignore
+    if mods:
+        t = "WARNING: write-access was removed from the following volumes because they are not mapped to an actual HDD for storage! All uploaded data would live in RAM only, and all uploaded files would be LOST on next reboot. To allow uploading and ignore this hazard, enable the 'wram' option (global/volflag). List of affected volumes:"
+        t2 = ["\n  volume=[/%s], abspath=%r, type=%s, root=%r" % x for x in mods]
+        log("vfs", t + "".join(t2) + "\n", 1)
+
+    assume = "mac" if MACOS else "win" if ANYWIN else "lin"
+    for vol in asrv.vfs.all_nodes.values():
+        if not vol.realpath or vol.flags.get("is_file"):
+            continue
+        zs = vol.flags["fsnt"].strip()[:3].lower()
+        if ANYWIN and not zs:
+            zs = "win"
+        if zs in ("lin", "win", "mac"):
+            vol.flags["fsnt"] = zs
+            continue
+        fs = _fstab.get(vol.realpath)[0]
+        fs = "win" if fs in winfs else assume
+        htm = json.loads(vol.js_htm)
+        vol.flags["fsnt"] = vol.js_ls["fsnt"] = htm["fsnt"] = fs
+        vol.js_htm = json_hesc(json.dumps(htm))
+        # md_htm is dontcare; only relevant for up2k request body
+
+
+def filefrag(ap: str) -> list[str]:
+    rc, so, se = runcmd([b"filefrag", b"-v", fsenc(ap)], timeout=10)
+    ptn = PTN_FILEFRAG
+    ret = []
+    for ln in so.split("\n"):
+        m = ptn.match(ln)
+        if not m:
+            continue
+        a, b, s = m.groups()
+        if "shared" in s.strip().split(","):
+            ret.append((a, b))
+    return ret

copyparty/ftpd.py

@@ -1,5 +1,5 @@
 # coding: utf-8
-from __future__ import print_function, unicode_literals
+from __future__ import division, print_function, unicode_literals
 
 import argparse
 import errno
@@ -15,20 +15,20 @@ from pyftpdlib.handlers import FTPHandler
 from pyftpdlib.ioloop import IOLoop
 from pyftpdlib.servers import FTPServer
 
-from .__init__ import PY2, TYPE_CHECKING
+from .__init__ import ANYWIN, PY2, TYPE_CHECKING
 from .authsrv import VFS
 from .bos import bos
 from .util import (
-    FN_EMB,
     VF_CAREFUL,
     Daemon,
     ODict,
     Pebkac,
     exclude_dotfiles,
+    exclude_dothidden,
     fsenc,
     ipnorm,
     pybin,
-    relchk,
+    read_utf8,
     runhook,
     sanitize_fn,
     set_fperms,
@@ -68,13 +68,13 @@ class FtpAuth(DummyAuthorizer):
         if ip.startswith("::ffff:"):
             ip = ip[7:]
 
-        ip = ipnorm(ip)
+        ipn = ipnorm(ip)
         bans = self.hub.bans
-        if ip in bans:
-            rt = bans[ip] - time.time()
+        if ipn in bans:
+            rt = bans[ipn] - time.time()
             if rt < 0:
                 logging.info("client unbanned")
-                del bans[ip]
+                del bans[ipn]
             else:
                 raise AuthenticationFailed("banned")
 
@@ -86,7 +86,7 @@ class FtpAuth(DummyAuthorizer):
             if args.usernames:
                 alts = ["%s:%s" % (username, password)]
             else:
-                alts = password, username
+                alts = [password, username]
 
             for zs in alts:
                 zs = asrv.iacct.get(asrv.ah.hash(zs), "")
@@ -148,19 +148,16 @@ class FtpFs(AbstractedFS):
         self.hub: "SvcHub" = cmd_channel.hub
         self.args = cmd_channel.args
         self.uname = cmd_channel.uname
+        self.log_src = "%s:FTP" % (cmd_channel.cli_ip,)
 
         self.cwd = "/"  # pyftpdlib convention of leading slash
         self.root = "/var/lib/empty"
 
-        self.can_read = self.can_write = self.can_move = False
-        self.can_delete = self.can_get = self.can_upget = False
-        self.can_admin = self.can_dot = False
-
         self.listdirinfo = self.listdir
         self.chdir(".")
 
-    def log(self, msg: str, c: Union[int, str] = 0) -> None:
-        self.hub.log("ftpd", msg, c)
+    def clog(self, msg: str, c: Union[int, str] = 0) -> None:
+        self.hub.log(self.log_src, msg, c)
 
     def v2a(
         self,
@@ -171,19 +168,26 @@ class FtpFs(AbstractedFS):
         d: bool = False,
     ) -> tuple[str, VFS, str]:
         try:
-            vpath = vpath.replace("\\", "/").strip("/")
-            rd, fn = os.path.split(vpath)
-            if relchk(rd):
-                logging.warning("malicious vpath: %s", vpath)
-                t = "Unsupported characters in [{}]"
-                raise FSE(t.format(vpath), 1)
+            # because:
+            #  * ftp-spec doesn't define sep
+            #  * windows filename limitations
+            # need to allow/translate illegal fn,
+            # but require filesystem-correct rd
+            if ANYWIN or ("\\" in vpath and "/" not in vpath):
+                vpath = vpath.replace("\\", "/")
+            vpath = vpath.strip("/\\")
+            if ANYWIN:
+                rd, fn = os.path.split(vpath)
+                fn = sanitize_fn(fn, "win")
+                vpath = vjoin(rd, fn)
+            # end of windows jank
 
-            fn = sanitize_fn(fn or "", "")
-            vpath = vjoin(rd, fn)
             vfs, rem = self.hub.asrv.vfs.get(vpath, self.uname, r, w, m, d)
+            vpath = vjoin(vfs.vpath, rem)
+            rd, fn = os.path.split(vpath)
             if (
                 w
-                and fn.lower() in FN_EMB
+                and fn.lower() in vfs.flags["emb_all"]
                 and self.h.uname not in vfs.axs.uread
                 and "wo_up_readme" not in vfs.flags
             ):
@@ -195,18 +199,12 @@ class FtpFs(AbstractedFS):
                 t = "No filesystem mounted at [{}]"
                 raise FSE(t.format(vpath))
 
-            if "xdev" in vfs.flags or "xvol" in vfs.flags:
-                ap = vfs.canonical(rem)
-                avfs = vfs.chk_ap(ap)
-                t = "Permission denied in [{}]"
-                if not avfs:
-                    raise FSE(t.format(vpath), 1)
+            ap = vfs.canonical(rem, False)
 
-                cr, cw, cm, cd, _, _, _, _ = avfs.can_access("", self.h.uname)
-                if r and not cr or w and not cw or m and not cm or d and not cd:
-                    raise FSE(t.format(vpath), 1)
+            if "bcasechk" in vfs.flags and not vfs.casechk(rem, True):
+                raise FSE("No such file or directory", 1)
 
-            return os.path.join(vfs.realpath, rem), vfs, rem
+            return ap, vfs, rem
         except Pebkac as ex:
             raise FSE(str(ex))
 
@@ -218,7 +216,7 @@ class FtpFs(AbstractedFS):
         m: bool = False,
         d: bool = False,
     ) -> tuple[str, VFS, str]:
-        return self.v2a(os.path.join(self.cwd, vpath), r, w, m, d)
+        return self.v2a(join(self.cwd, vpath), r, w, m, d)
 
     def ftp2fs(self, ftppath: str) -> str:
         # return self.v2a(ftppath)
@@ -236,6 +234,8 @@ class FtpFs(AbstractedFS):
         return True
 
     def open(self, filename: str, mode: str) -> typing.IO[Any]:
+        self.clog("open(%r,%r) @%s" % (filename, mode, self.uname))
+
         r = "r" in mode
         w = "w" in mode or "a" in mode or "+" in mode
 
@@ -250,9 +250,36 @@ class FtpFs(AbstractedFS):
                 need_unlink = False
                 td = 0
 
-        if w and need_unlink:
+            xbu = vfs.flags.get("xbu")
+            if xbu:
+                hr = runhook(
+                    self.clog,
+                    None,
+                    self.hub.up2k,
+                    "xbu.ftp",
+                    xbu,
+                    ap,
+                    filename,
+                    "",
+                    "",
+                    "",
+                    0,
+                    0,
+                    "1.3.8.7",
+                    time.time(),
+                    None,
+                )
+                t = hr.get("rejectmsg") or ""
+                if t or hr.get("rc") != 0:
+                    if not t:
+                        t = "upload blocked by xbu server config: %r" % (filename,)
+                    self.clog(t, 3)
+                    raise FSE(t)
+
+        if w and need_unlink:  # type: ignore  # !rm
+            assert td  # type: ignore  # !rm
             if td >= -1 and td <= self.args.ftp_wt:
-                # within permitted timeframe; unlink and accept
+                # within permitted timeframe; allow overwrite or resume
                 do_it = True
             elif self.args.no_del or self.args.ftp_no_ow:
                 # file too old, or overwrite not allowed; reject
@@ -269,17 +296,28 @@ class FtpFs(AbstractedFS):
             if not do_it:
                 raise FSE("File already exists")
 
-            wunlink(self.log, ap, VF_CAREFUL)
+            # Don't unlink file for append mode
+            elif "a" not in mode:
+                wunlink(self.clog, ap, VF_CAREFUL)
 
         ret = open(fsenc(ap), mode, self.args.iobuf)
         if w and "fperms" in vfs.flags:
             set_fperms(ret, vfs.flags)
 
+        self.clog(" `-- %r" % (ap,))
         return ret
 
     def chdir(self, path: str) -> None:
         nwd = join(self.cwd, path)
-        vfs, rem = self.hub.asrv.vfs.get(nwd, self.uname, False, False)
+        try:
+            vfs, rem = self.hub.asrv.vfs.get(nwd, self.uname, False, False)
+        except Exception as ex:
+            raise FSE(str(ex))
+
+        if not vfs.realpath:
+            self.cwd = nwd
+            return
+
         ap = vfs.canonical(rem)
         try:
             st = bos.stat(ap)
@@ -289,30 +327,19 @@ class FtpFs(AbstractedFS):
             # returning 550 is library-default and suitable
             raise FSE("No such file or directory")
 
-        if vfs.realpath:
-            avfs = vfs.chk_ap(ap, st)
-            if not avfs:
-                raise FSE("Permission denied", 1)
-        else:
-            avfs = vfs
+        avfs = vfs.chk_ap(ap, st)
+        if not avfs:
+            raise FSE("Permission denied", 1)
 
         self.cwd = nwd
-        (
-            self.can_read,
-            self.can_write,
-            self.can_move,
-            self.can_delete,
-            self.can_get,
-            self.can_upget,
-            self.can_admin,
-            self.can_dot,
-        ) = avfs.can_access("", self.h.uname)
 
     def mkdir(self, path: str) -> None:
+        self.clog("mkdir(%r) @%s" % (path, self.uname))
         ap, vfs, _ = self.rv2a(path, w=True)
         bos.makedirs(ap, vf=vfs.flags)  # filezilla expects this
 
     def listdir(self, path: str) -> list[str]:
+        self.clog("ls(%r) @%s" % (path, self.uname))
         vpath = join(self.cwd, path)
         try:
             ap, vfs, rem = self.v2a(vpath, True, False)
@@ -329,8 +356,11 @@ class FtpFs(AbstractedFS):
             vfs_ls = [x[0] for x in vfs_ls1]
             vfs_ls.extend(vfs_virt.keys())
 
-            if not self.can_dot:
-                vfs_ls = exclude_dotfiles(vfs_ls)
+            if self.uname not in vfs.axs.udot:
+                if "dothidden" in vfs.flags and ".hidden" in [x[0] for x in vfs_ls]:
+                    vfs_ls = exclude_dothidden(vfs_ls, fsroot)
+                else:
+                    vfs_ls = exclude_dotfiles(vfs_ls)
 
             vfs_ls.sort()
             return vfs_ls
@@ -359,6 +389,7 @@ class FtpFs(AbstractedFS):
             return ret
 
     def rmdir(self, path: str) -> None:
+        self.clog("rmdir(%r) @%s" % (path, self.uname))
         ap = self.rv2a(path, d=True)[0]
         try:
             bos.rmdir(ap)
@@ -367,6 +398,7 @@ class FtpFs(AbstractedFS):
                 raise
 
     def remove(self, path: str) -> None:
+        self.clog("rm(%r) @%s" % (path, self.uname))
         if self.args.no_del:
             raise FSE("The delete feature is disabled in server config")
 
@@ -377,16 +409,14 @@ class FtpFs(AbstractedFS):
             raise FSE(str(ex))
 
     def rename(self, src: str, dst: str) -> None:
-        if not self.can_move:
-            raise FSE("Not allowed for user " + self.h.uname)
-
+        self.clog("mv(%r,%r) @%s" % (src, dst, self.uname))
         if self.args.no_mv:
             raise FSE("The rename/move feature is disabled in server config")
 
         svp = join(self.cwd, src).lstrip("/")
         dvp = join(self.cwd, dst).lstrip("/")
         try:
-            self.hub.up2k.handle_mv(self.uname, self.h.cli_ip, svp, dvp)
+            self.hub.up2k.handle_mv("", self.uname, self.h.cli_ip, svp, dvp)
         except Exception as ex:
             raise FSE(str(ex))
 
@@ -409,17 +439,17 @@ class FtpFs(AbstractedFS):
             return st
 
     def utime(self, path: str, timeval: float) -> None:
-        try:
-            ap = self.rv2a(path, w=True)[0]
-            return bos.utime(ap, (int(time.time()), int(timeval)))
-        except Exception as ex:
-            logging.error("ftp.utime: %s, %r", ex, ex)
-            raise
+        self.clog("utime(%r,%r) @%s" % (path, timeval, self.uname))
+        ap = self.rv2a(path, w=True)[0]
+        bos.utime_c(logging.warning, ap, int(timeval), False)
 
     def lstat(self, path: str) -> os.stat_result:
         ap = self.rv2a(path)[0]
         return bos.stat(ap)
 
+    def readlink(self, *a) -> None:
+        raise OSError(errno.EINVAL, "nope")
+
     def isfile(self, path: str) -> bool:
         try:
             st = self.stat(path)
@@ -462,9 +492,13 @@ class FtpFs(AbstractedFS):
     def get_user_by_uid(self, uid: int) -> str:
         return "root"
 
-    def get_group_by_uid(self, gid: int) -> str:
+    def get_group_by_gid(self, gid: int) -> str:
         return "root"
 
+    def mkstemp(self, *a, **ka):
+        self.clog("mkstemp%r @%s" % (a, self.uname))
+        raise FSE("nope")
+
 
 class FtpHandler(FTPHandler):
     abstracted_fs = FtpFs
@@ -493,6 +527,7 @@ class FtpHandler(FTPHandler):
             return
 
         self.cli_ip = cip
+        self.log_src = "%s:FTP" % (cip,)
 
         # abspath->vpath mapping to resolve log_transfer paths
         self.vfs_map: dict[str, str] = {}
@@ -500,6 +535,15 @@ class FtpHandler(FTPHandler):
         # reduce non-debug logging
         self.log_cmds_list = [x for x in self.log_cmds_list if x not in ("CWD", "XCWD")]
 
+    def clog(self, msg: str, c: Union[int, str] = 0) -> None:
+        self.hub.log(self.log_src, msg, c)
+
+    def ftp_NOOP(self, *a) -> None:
+        self.respond("200 pong", logging.debug)
+
+    def ftp_STOU(self, *a) -> None:
+        self.respond("550 nope", logging.warning)
+
     def ftp_STOR(self, file: str, mode: str = "w") -> Any:
         # Optional[str]
         vp = join(self.fs.cwd, file).lstrip("/")
@@ -510,24 +554,30 @@ class FtpHandler(FTPHandler):
             return
         self.vfs_map[ap] = vp
         xbu = vfs.flags.get("xbu")
-        if xbu and not runhook(
-            None,
-            None,
-            self.hub.up2k,
-            "xbu.ftpd",
-            xbu,
-            ap,
-            vp,
-            "",
-            self.uname,
-            self.hub.asrv.vfs.get_perms(vp, self.uname),
-            0,
-            0,
-            self.cli_ip,
-            time.time(),
-            "",
-        ):
-            raise FSE("Upload blocked by xbu server config")
+        if xbu:
+            hr = runhook(
+                self.clog,
+                None,
+                self.hub.up2k,
+                "xbu.ftpd",
+                xbu,
+                ap,
+                vp,
+                "",
+                self.uname,
+                self.hub.asrv.vfs.get_perms(vp, self.uname),
+                0,
+                0,
+                self.cli_ip,
+                time.time(),
+                None,
+            )
+            t = hr.get("rejectmsg") or ""
+            if t or hr.get("rc") != 0:
+                if not t:
+                    t = "Upload blocked by xbu server config: %r" % (vp,)
+                self.respond("550 %s" % (t,), logging.info)
+                return
 
         # print("ftp_STOR: {} {} => {}".format(vp, mode, ap))
         ret = FTPHandler.ftp_STOR(self, file, mode)
@@ -593,17 +643,25 @@ class Ftpd(object):
                 print(t.format(pybin))
                 sys.exit(1)
 
+            if self.args.certkey:
+                h1.keyfile = self.args.certkey
             h1.certfile = self.args.cert
             h1.tls_control_required = True
             h1.tls_data_required = True
 
             hs.append([h1, self.args.ftps])
 
+        zs = self.args.ftp_banner.replace("\\n", "\n")
+        if zs.startswith("@"):
+            zs = read_utf8(None, zs[1:], False)
+        banner = zs.replace("\r", "").replace("\n", "\r\n").strip()
+
         for h_lp in hs:
             h2, lp = h_lp
             FtpHandler.hub = h2.hub = hub
             FtpHandler.args = h2.args = hub.args
             FtpHandler.authorizer = h2.authorizer = FtpAuth(hub)
+            FtpHandler.banner = banner
 
             if self.args.ftp_pr:
                 p1, p2 = [int(x) for x in self.args.ftp_pr.split("-")]
@@ -623,7 +681,7 @@ class Ftpd(object):
         lgr = logging.getLogger("pyftpdlib")
         lgr.setLevel(logging.DEBUG if self.args.ftpv else logging.INFO)
 
-        ips = self.args.i
+        ips = self.args.ftp_i
         if "::" in ips:
             ips.append("0.0.0.0")
 

copyparty/httpconn.py

@@ -1,32 +1,29 @@
 # coding: utf-8
-from __future__ import print_function, unicode_literals
+from __future__ import division, print_function, unicode_literals
 
 import argparse  # typechk
-import os
 import re
 import socket
 import threading  # typechk
 import time
 
-try:
-    if os.environ.get("PRTY_NO_TLS"):
-        raise Exception()
-
-    HAVE_SSL = True
-    import ssl
-except:
-    HAVE_SSL = False
-
 from . import util as Util
 from .__init__ import TYPE_CHECKING, EnvParams
 from .authsrv import AuthSrv  # typechk
+from .cert import NO_TLS
 from .httpcli import HttpCli
-from .ico import Ico
-from .mtag import HAVE_FFMPEG
-from .th_cli import ThumbCli
-from .th_srv import HAVE_PIL, HAVE_VIPS
 from .u2idx import U2idx
-from .util import HMaccas, NetMap, shut_socket
+from .util import HMaccas, NetMap, min_ex, shut_socket
+
+try:
+    if NO_TLS:
+        raise Exception()
+
+    import ssl
+
+    HAVE_SSL = True
+except:
+    HAVE_SSL = False
 
 if True:  # pylint: disable=using-constant-test
     from typing import Optional, Pattern, Union
@@ -62,27 +59,22 @@ class HttpConn(object):
         self.ipu_iu: Optional[dict[str, str]] = hsrv.ipu_iu
         self.ipu_nm: Optional[NetMap] = hsrv.ipu_nm
         self.ipa_nm: Optional[NetMap] = hsrv.ipa_nm
+        self.ipar_nm: Optional[NetMap] = hsrv.ipar_nm
         self.xff_nm: Optional[NetMap] = hsrv.xff_nm
         self.xff_lan: NetMap = hsrv.xff_lan  # type: ignore
         self.iphash: HMaccas = hsrv.broker.iphash
         self.bans: dict[str, int] = hsrv.bans
         self.aclose: dict[str, int] = hsrv.aclose
 
-        enth = (HAVE_PIL or HAVE_VIPS or HAVE_FFMPEG) and not self.args.no_thumb
-        self.thumbcli: Optional[ThumbCli] = ThumbCli(hsrv) if enth else None  # mypy404
-        self.ico: Ico = Ico(self.args)  # mypy404
-
         self.t0: float = time.time()  # mypy404
         self.freshen_pwd: float = 0.0
         self.stopping = False
         self.nreq: int = -1  # mypy404
         self.nbyte: int = 0  # mypy404
         self.u2idx: Optional[U2idx] = None
+        self.lf_url: Optional[Pattern[str]] = self.args.lf_url
         self.log_func: "Util.RootLogger" = hsrv.log  # mypy404
         self.log_src: str = "httpconn"  # mypy404
-        self.lf_url: Optional[Pattern[str]] = (
-            re.compile(self.args.lf_url) if self.args.lf_url else None
-        )  # mypy404
         self.set_rproxy()
 
     def shutdown(self) -> None:
@@ -166,7 +158,7 @@ class HttpConn(object):
             try:
                 assert ssl  # type: ignore  # !rm
                 ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
-                ctx.load_cert_chain(self.args.cert)
+                ctx.load_cert_chain(self.args.cert, self.args.certkey)
                 if self.args.ssl_ver:
                     ctx.options &= ~self.args.ssl_flags_en
                     ctx.options |= self.args.ssl_flags_de
@@ -203,12 +195,12 @@ class HttpConn(object):
             except Exception as ex:
                 em = str(ex)
 
-                if "ALERT_CERTIFICATE_UNKNOWN" in em:
-                    # android-chrome keeps doing this
-                    pass
+                if "ALERT_" in em:
+                    self.log("client refused our TLS cert or config: " + em, c=6)
 
                 else:
-                    self.log("handshake\033[0m " + em, c=5)
+                    t = "https-handshake failed, probably due to client:\n"
+                    self.log(t + min_ex(), c=5)
 
                 return
 
@@ -221,6 +213,21 @@ class HttpConn(object):
             if not self.cli.run():
                 return
 
+            if self.sr.te == 1:
+                self.log("closing socket (leftover TE)", "90")
+                return
+
+            if (
+                "content-length" in self.cli.headers
+                and int(self.cli.headers["content-length"]) != self.sr.nb
+            ):
+                self.log("closing socket (CL mismatch)", "90")
+                return
+
+            # note: proxies reject PUT sans Content-Length; illegal for HTTP/1.1
+
+            self.sr.nb = self.sr.te = 0
+
             if self.u2idx:
                 self.hsrv.put_u2idx(str(self.addr), self.u2idx)
                 self.u2idx = None

copyparty/httpsrv.py

@@ -1,8 +1,7 @@
 # coding: utf-8
-from __future__ import print_function, unicode_literals
+from __future__ import division, print_function, unicode_literals
 
 import hashlib
-import math
 import os
 import re
 import socket
@@ -55,8 +54,14 @@ except SyntaxError:
     )
     sys.exit(1)
 
+from .authsrv import LEELOO_DALLAS
 from .httpconn import HttpConn
+from .ico import Ico
 from .metrics import Metrics
+from .mtag import HAVE_FFMPEG
+from .sutil import gfilter2
+from .th_cli import ThumbCli
+from .th_srv import HAVE_PIL, HAVE_VIPS
 from .u2idx import U2idx
 from .util import (
     E_SCK,
@@ -133,16 +138,20 @@ class HttpSrv(object):
         self.bans: dict[str, int] = {}
         self.aclose: dict[str, int] = {}
 
+        self.thumbcli: Optional[ThumbCli] = None
+        self.ico: Ico = Ico(self.args)
+
         dli: dict[str, tuple[float, int, "VFS", str, str]] = {}  # info
         dls: dict[str, tuple[float, int]] = {}  # state
         self.dli = self.tdli = dli
         self.dls = self.tdls = dls
-        self.iiam = '<img src="%s.cpr/iiam.gif?cache=i" />' % (self.args.SRS,)
+        self.iiam = '<img src="%s.cpr/w/iiam.gif?cache=i" />' % (self.args.SRS,)
 
         self.bound: set[tuple[str, int]] = set()
         self.name = "hsrv" + nsuf
         self.mutex = threading.Lock()
         self.u2mutex = threading.Lock()
+        self.bad_ver = False
         self.stopping = False
 
         self.tp_nthr = 0  # actual
@@ -170,6 +179,8 @@ class HttpSrv(object):
         self.u2idx_free: dict[str, U2idx] = {}
         self.u2idx_n = 0
 
+        self.wopi_files: dict[str, dict[str, Any]] = {}
+
         assert jinja2  # type: ignore  # !rm
         env = jinja2.Environment()
         env.loader = jinja2.FunctionLoader(lambda f: load_jinja2_resource(self.E, f))
@@ -185,8 +196,11 @@ class HttpSrv(object):
             "shares",
             "splash",
             "svcs",
+            "wopi",
         ]
         self.j2 = {x: env.get_template(x + ".html") for x in jn}
+        self.j2["opds"] = env.get_template("opds.xml")
+        self.j2["opds_osd"] = env.get_template("opds_osd.xml")
         self.prism = has_resource(self.E, "web/deps/prism.js.gz")
 
         if self.args.ipu:
@@ -200,6 +214,7 @@ class HttpSrv(object):
             self.ipr = None
 
         self.ipa_nm = build_netmap(self.args.ipa)
+        self.ipar_nm = build_netmap(self.args.ipar)
         self.xff_nm = build_netmap(self.args.xff_src)
         self.xff_lan = build_netmap("lan")
 
@@ -226,16 +241,23 @@ class HttpSrv(object):
             if self.args.log_thrs:
                 start_log_thrs(self.log, self.args.log_thrs, nid)
 
-        self.th_cfg: dict[str, set[str]] = {}
-        Daemon(self.post_init, "hsrv-init2")
+        if (HAVE_PIL or HAVE_VIPS or HAVE_FFMPEG) and not self.args.no_thumb:
+            Daemon(self.post_init, "hsrv-init2")
 
     def post_init(self) -> None:
         try:
             x = self.broker.ask("thumbsrv.getcfg")
-            self.th_cfg = x.get()
+            self.thumbcli = ThumbCli(self, x.get())
         except:
             pass
 
+    def set_th_cfg(self, c: dict[str, set[str]], opts: tuple[bool]) -> None:
+        self.args.th_no_jxl = opts[0]
+        self.thumbcli = ThumbCli(self, c)
+
+    def set_bad_ver(self) -> None:
+        self.bad_ver = True
+
     def set_netdevs(self, netdevs: dict[str, Netdev]) -> None:
         ips = set()
         for ip, _ in self.bound:
@@ -295,7 +317,7 @@ class HttpSrv(object):
 
         self.srvs.append(sck)
         self.bound.add((ip, port))
-        self.nclimax = math.ceil(self.args.nc * 1.0 / nlisteners)
+        self.nclimax = -int(-self.args.nc // nlisteners)
         Daemon(
             self.thr_listen,
             "httpsrv-n{}-listen-{}-{}".format(self.nid or "0", ip, port),
@@ -382,8 +404,8 @@ class HttpSrv(object):
                 if nloris < nconn / 2:
                     continue
 
-                t = "slowloris (idle-conn): {} banned for {} min"
-                self.log(self.name, t.format(ip, self.args.loris, nclose), 1)
+                t = "slow%s (idle-conn): %s banned for %d min"  # slowloris
+                self.log(self.name, t % ("loris", ip, self.args.loris), 1)
                 self.bans[ip] = int(time.time() + self.args.loris * 60)
 
             if self.args.log_conn:
@@ -571,7 +593,7 @@ class HttpSrv(object):
 
             v = self.E.t0
             try:
-                with os.scandir(os.path.join(self.E.mod, "web")) as dh:
+                with os.scandir(self.E.mod_ + "web") as dh:
                     for fh in dh:
                         inf = fh.stat()
                         v = max(v, inf.st_mtime)
@@ -639,3 +661,73 @@ class HttpSrv(object):
 
         self.tdli = dli
         self.tdls = dls
+
+    def pregen_thumbs(self) -> None:
+        Daemon(self._pregen_thumbs, "th_pregen")
+
+    def _pregen_thumbs(self) -> None:
+        def log(msg, n):
+            self.log("thumb-pregen", msg, n)
+
+        if getattr(self, "_pregen", False):
+            log("already running", 1)
+            return
+
+        self._pregen = True
+
+        for n in range(9999999):
+            x = self.broker.ask("up2k.is_busy")
+            zb, zi = x.get()
+            if zi:
+                break
+            if not n:
+                log("waiting for up2k to finish initializing", 6)
+            time.sleep(1 if n < 10 else 5 if n < 300 else 15)
+
+        if not self.thumbcli:
+            log("no thumbcli", 1)
+            return
+
+        if self.args.th_pre_rl:
+            try:
+                self.broker.hub.thumbsrv.log = self.broker.hub.thumbsrv._slog
+            except:
+                pass
+
+        nfiles = 0
+        t0 = time.time()
+        scandir = not self.args.no_scandir
+        for vn in self.asrv.vfs.all_nodes.values():
+            fmts = vn.flags.get("th_pregen", "")
+            if not fmts:
+                continue
+            log("starting for volume /%s" % (vn.vpath,), 6)
+            g = vn.walk("", "/", [], LEELOO_DALLAS, [[True]], 2, scandir, False, False)
+            g = gfilter2(g, self, vn.vpath, fmts.split(","))
+            for f in g:
+                nfiles += 1
+                if not nfiles % 256:
+                    now = time.time()
+                    for n in range(9999999):
+                        x = self.broker.ask("up2k.is_busy")
+                        zb, zi = x.get()
+                        if not zb:
+                            if n:
+                                t0 += time.time() - now
+                            break
+                        if not n:
+                            log("waiting for up2k to finish indexing", 6)
+                        time.sleep(5)
+
+        if self.args.th_pre_rl:
+            try:
+                self.broker.hub.thumbsrv.log = self.broker.hub.thumbsrv._log
+            except:
+                pass
+
+        t = "finished; %d files in %d seconds"
+        log(t % (nfiles, time.time() - t0), 6)
+        self._pregen = False
+
+        if self.args.exit == "thgen":
+            self.broker.say("sigterm")

copyparty/ico.py

@@ -1,5 +1,5 @@
 # coding: utf-8
-from __future__ import print_function, unicode_literals
+from __future__ import division, print_function, unicode_literals
 
 import argparse  # typechk
 import colorsys

copyparty/mdns.py

@@ -1,7 +1,8 @@
 # coding: utf-8
-from __future__ import print_function, unicode_literals
+from __future__ import division, print_function, unicode_literals
 
 import errno
+import os
 import random
 import select
 import socket
@@ -12,22 +13,51 @@ from ipaddress import IPv4Network, IPv6Network
 from .__init__ import TYPE_CHECKING
 from .__init__ import unicode as U
 from .multicast import MC_Sck, MCast
-from .stolen.dnslib import AAAA
-from .stolen.dnslib import CLASS as DC
-from .stolen.dnslib import (
-    NSEC,
-    PTR,
-    QTYPE,
-    RR,
-    SRV,
-    TXT,
-    A,
-    DNSHeader,
-    DNSQuestion,
-    DNSRecord,
-    set_avahi_379,
-)
-from .util import CachedSet, Daemon, Netdev, list_ips, min_ex
+from .util import IP6_LL, CachedSet, Daemon, Netdev, list_ips, min_ex
+
+try:
+    if os.environ.get("PRTY_SYS_ALL") or os.environ.get("PRTY_SYS_DNSLIB"):
+        raise ImportError()
+    from .stolen.dnslib import (
+        AAAA,
+    )
+    from .stolen.dnslib import CLASS as DC
+    from .stolen.dnslib import (
+        NSEC,
+        PTR,
+        QTYPE,
+        RR,
+        SRV,
+        TXT,
+        A,
+        DNSHeader,
+        DNSQuestion,
+        DNSRecord,
+        set_avahi_379,
+    )
+
+    DNS_VND = True
+except ImportError:
+    DNS_VND = False
+    from dnslib import (
+        AAAA,
+    )
+    from dnslib import CLASS as DC
+    from dnslib import (
+        NSEC,
+        PTR,
+        QTYPE,
+        RR,
+        SRV,
+        TXT,
+        A,
+        DNSHeader,
+        DNSQuestion,
+        DNSRecord,
+    )
+
+    DC.forward[0x8001] = "F_IN"
+    DC.reverse["F_IN"] = 0x8001
 
 if TYPE_CHECKING:
     from .svchub import SvcHub
@@ -35,6 +65,11 @@ if TYPE_CHECKING:
 if True:  # pylint: disable=using-constant-test
     from typing import Any, Optional, Union
 
+if os.environ.get("PRTY_MODSPEC"):
+    from inspect import getsourcefile
+
+    print("PRTY_MODSPEC: dnslib:", getsourcefile(A))
+
 
 MDNS4 = "224.0.0.251"
 MDNS6 = "ff02::fb"
@@ -73,8 +108,8 @@ class MDNS(MCast):
         self.ngen = ngen
         self.ttl = 300
 
-        if not self.args.zm_nwa_1:
-            set_avahi_379()
+        if not self.args.zm_nwa_1 and DNS_VND:
+            set_avahi_379()  # type: ignore
 
         zs = self.args.zm_fqdn or (self.args.name + ".local")
         zs = zs.replace("--name", self.args.name).rstrip(".") + "."
@@ -100,9 +135,14 @@ class MDNS(MCast):
         self.log_func(self.logsrc, msg, c)
 
     def build_svcs(self) -> tuple[dict[str, dict[str, Any]], set[str]]:
+        ar = self.args
         zms = self.args.zms
-        http = {"port": 80 if 80 in self.args.p else self.args.p[0]}
-        https = {"port": 443 if 443 in self.args.p else self.args.p[0]}
+
+        zi = ar.zm_http
+        http = {"port": zi if zi != -1 else 80 if 80 in ar.p else ar.p[0]}
+        zi = ar.zm_https
+        https = {"port": zi if zi != -1 else 443 if 443 in ar.p else ar.p[0]}
+
         webdav = http.copy()
         webdavs = https.copy()
         webdav["u"] = webdavs["u"] = "u"  # KDE requires username
@@ -127,16 +167,16 @@ class MDNS(MCast):
 
         svcs: dict[str, dict[str, Any]] = {}
 
-        if "d" in zms:
+        if "d" in zms and http["port"]:
             svcs["_webdav._tcp.local."] = webdav
 
-        if "D" in zms:
+        if "D" in zms and https["port"]:
             svcs["_webdavs._tcp.local."] = webdavs
 
-        if "h" in zms:
+        if "h" in zms and http["port"]:
             svcs["_http._tcp.local."] = http
 
-        if "H" in zms:
+        if "H" in zms and https["port"]:
             svcs["_https._tcp.local."] = https
 
         if "f" in zms.lower():
@@ -375,7 +415,7 @@ class MDNS(MCast):
         cip = addr[0]
         v6 = ":" in cip
         if (cip.startswith("169.254") and not self.ll_ok) or (
-            v6 and not cip.startswith("fe80")
+            v6 and not cip.startswith(IP6_LL)
         ):
             return
 

copyparty/metrics.py

@@ -1,5 +1,5 @@
 # coding: utf-8
-from __future__ import print_function, unicode_literals
+from __future__ import division, print_function, unicode_literals
 
 import json
 import time
@@ -17,10 +17,10 @@ class Metrics(object):
         self.hsrv = hsrv
 
     def tx(self, cli: "HttpCli") -> bool:
-        if not cli.avol:
+        args = cli.args
+        if not cli.avol and cli.uname.lower() not in args.stats_u_set:
             raise Pebkac(403, "'stats' not allowed for user " + cli.uname)
 
-        args = cli.args
         if not args.stats:
             raise Pebkac(403, "the stats feature is not enabled in server config")
 

copyparty/mtag.py

@@ -1,5 +1,5 @@
 # coding: utf-8
-from __future__ import print_function, unicode_literals
+from __future__ import division, print_function, unicode_literals
 
 import argparse
 import json
@@ -15,8 +15,12 @@ from .authsrv import VFS
 from .bos import bos
 from .util import (
     FFMPEG_URL,
+    LOG,
     REKOBO_LKEY,
+    SCWD,
     VF_CAREFUL,
+    absreal,
+    expand_osenv_c,
     fsenc,
     gzip,
     min_ex,
@@ -26,6 +30,9 @@ from .util import (
     sfsenc,
     uncyg,
     wunlink,
+    zip_fi,
+    zip_lim,
+    zip_readf,
 )
 
 if True:  # pylint: disable=using-constant-test
@@ -45,29 +52,42 @@ except:
     HAVE_MUTAGEN = False
 
 
-def have_ff(scmd: str) -> bool:
-    if ANYWIN:
+def have_ff(name: str) -> list[bytes]:
+    uname = name.upper()
+    if os.environ.get("PRTY_NO_" + uname):
+        return []
+
+    ebin = os.environ.get("PRTY_%s_BIN" % (uname,))
+    try:
+        scmd = (ebin or name).decode("utf-8")
+    except:
+        scmd: str = ebin or name
+
+    if ANYWIN and not ebin:
         scmd += ".exe"
 
     if PY2:
-        print("# checking {}".format(scmd))
-        acmd = (scmd + " -version").encode("ascii").split(b" ")
+        print("# checking %s" % (scmd,))
+        bcmd = scmd.encode("utf-8")
         try:
-            sp.Popen(acmd, stdout=sp.PIPE, stderr=sp.PIPE).communicate()
-            return True
+            sp.Popen([bcmd, b"-version"], stdout=sp.PIPE, stderr=sp.PIPE).communicate()
+            return [bcmd]
         except:
-            return False
+            return []
     else:
-        return bool(shutil.which(scmd))
+        ret = shutil.which(scmd)
+        return [absreal(ret).encode("utf-8")] if ret else []
 
 
-HAVE_FFMPEG = not os.environ.get("PRTY_NO_FFMPEG") and have_ff("ffmpeg")
-HAVE_FFPROBE = not os.environ.get("PRTY_NO_FFPROBE") and have_ff("ffprobe")
+HAVE_FFMPEG = have_ff("ffmpeg")
+HAVE_FFPROBE = have_ff("ffprobe")
+TH_BWRAP = []
 
-CBZ_PICS = set("png jpg jpeg gif bmp tga tif tiff webp avif".split())
+CBZ_PICS = set("png jpg jpeg gif bmp tga tif tiff webp avif jxl".split())
 CBZ_01 = re.compile(r"(^|[^0-9v])0+[01]\b")
 
 FMT_AU = set("mp3 ogg flac wav".split())
+M4A = set("aac m4a m4b m4r".split())
 
 
 class MParser(object):
@@ -85,7 +105,7 @@ class MParser(object):
 
         while True:
             try:
-                bp = os.path.expanduser(args)
+                bp = os.path.expanduser(expand_osenv_c(args))
                 if WINDOWS:
                     bp = uncyg(bp)
 
@@ -132,6 +152,7 @@ class MParser(object):
 def au_unpk(
     log: "NamedLogger", fmt_map: dict[str, str], abspath: str, vn: Optional[VFS] = None
 ) -> str:
+    fd = 0
     ret = ""
     maxsz = 1024 * 1024 * 64
     try:
@@ -156,7 +177,7 @@ def au_unpk(
             zil = [x for x in zil if x.filename.lower().split(".")[-1] == au]
             if not zil:
                 raise Exception("no audio inside zip")
-            fi = zf.open(zil[0])
+            fi = zf.open(zip_lim(zil[0], maxsz))
 
         elif pk == "cbz":
             import zipfile
@@ -168,22 +189,31 @@ def au_unpk(
             znil = [x for x in znil if "cover" in x[0]] or znil
             znil = [x for x in znil if CBZ_01.search(x[0])] or znil
             t = "cbz: %d files, %d hits" % (nf, len(znil))
-            using = sorted(znil)[0][1].filename
-            if znil:
-                t += ", using " + using
-            log(t)
             if not znil:
                 raise Exception("no images inside cbz")
-            fi = zf.open(using)
+            zi = sorted(znil)[0][1]
+            if znil:
+                t += ", using " + zi.filename
+            log(t)
+            fi = zf.open(zip_lim(zi, maxsz))
+
+        elif pk == "kra" or pk == "ora":
+            import zipfile
+
+            zf = zipfile.ZipFile(abspath, "r")
+            zi = zip_fi(zf, "mergedimage.png", maxsz)
+            fi = zf.open(zi)
 
         elif pk == "epub":
             fi = get_cover_from_epub(log, abspath)
+            assert fi  # !rm
 
         else:
             raise Exception("unknown compression %s" % (pk,))
 
         fsz = 0
         with os.fdopen(fd, "wb") as fo:
+            fd = 0
             while True:
                 buf = fi.read(32768)
                 if not buf:
@@ -198,26 +228,48 @@ def au_unpk(
         return ret
 
     except Exception as ex:
+        if fd:
+            os.close(fd)
         if ret:
-            t = "failed to decompress audio file %r: %r"
+            t = "failed to decompress file %r: %r"
             log(t % (abspath, ex))
             wunlink(log, ret, vn.flags if vn else VF_CAREFUL)
+            return ""
 
         return abspath
 
 
+def bwrap(prog: bytes, ap_in: bytes, ap_out: bytes) -> list[bytes]:
+    if not TH_BWRAP:
+        return [prog]
+    ret = TH_BWRAP + [b"--ro-bind", prog, prog, b"--ro-bind", ap_in, ap_in]
+    if ap_out:
+        zs = ap_out.rsplit(b"/", 1)[0]
+        ret += [b"--bind", zs, zs]
+    ret.append(prog)
+    return ret
+
+
+def bwrap_fail(serr: str) -> None:
+    if "bwrap:" in serr:
+        LOG[0]("root", "ffmpeg failed due to --th-bwrap;\n  " + serr, 3)
+
+
 def ffprobe(
     abspath: str, timeout: int = 60
 ) -> tuple[dict[str, tuple[int, Any]], dict[str, list[Any]], list[Any], dict[str, Any]]:
-    cmd = [
-        b"ffprobe",
+    # ffprobe -hide_banner -show_streams -show_format --
+    bap = fsenc(abspath)
+    cmd = bwrap(HAVE_FFPROBE[0], bap, b"") + [
         b"-hide_banner",
         b"-show_streams",
         b"-show_format",
         b"--",
-        fsenc(abspath),
+        bap,
     ]
-    rc, so, se = runcmd(cmd, timeout=timeout, nice=True, oom=200)
+    rc, so, se = runcmd(cmd, cwd=SCWD, timeout=timeout, nice=True, oom=200)
+    if rc and TH_BWRAP:
+        bwrap_fail(se)
     retchk(rc, cmd, se)
     return parse_ffprobe(so)
 
@@ -257,7 +309,7 @@ def parse_ffprobe(
     md: dict[str, list[Any]] = {}  # raw tags
 
     is_audio = fmt.get("format_name") in FMT_AU
-    if fmt.get("filename", "").split(".")[-1].lower() in ["m4a", "aac"]:
+    if fmt.get("filename", "").split(".")[-1].lower() in M4A:
         is_audio = True
 
     # if audio file, ensure audio stream appears first
@@ -383,14 +435,15 @@ def get_cover_from_epub(log: "NamedLogger", abspath: str) -> Optional[IO[bytes]]
     from .dxml import parse_xml
 
     try:
-        from urlparse import urljoin  # Python2
+        from urlparse import urljoin  # type: ignore  # Python2
     except ImportError:
         from urllib.parse import urljoin  # Python3
 
     with zipfile.ZipFile(abspath, "r") as z:
         # First open the container file to find the package document (.opf file)
         try:
-            container_root = parse_xml(z.read("META-INF/container.xml").decode())
+            zb = zip_readf(z, "META-INF/container.xml", 2 << 20)
+            container_root = parse_xml(zb.decode())
         except KeyError:
             log("epub: no container file found in %s" % (abspath,))
             return None
@@ -404,7 +457,8 @@ def get_cover_from_epub(log: "NamedLogger", abspath: str) -> Optional[IO[bytes]]
 
         # Then open the first package document to find the path of the cover image
         try:
-            package_root = parse_xml(z.read(rootfile_path).decode())
+            zb = zip_readf(z, rootfile_path, 2 << 20)
+            package_root = parse_xml(zb.decode())
         except KeyError:
             log("epub: no package document found in %s" % (abspath,))
             return None
@@ -421,10 +475,18 @@ def get_cover_from_epub(log: "NamedLogger", abspath: str) -> Optional[IO[bytes]]
             # This might be an EPUB2 file, try the legacy way of specifying covers
             coverimage_path = _get_cover_from_epub2(log, package_root, package_ns)
 
+        if not coverimage_path:
+            raise Exception("no cover inside epub")
+
         # This url is either absolute (in the .epub) or relative to the package document
         adjusted_cover_path = urljoin(rootfile_path, coverimage_path)
 
-        return z.open(adjusted_cover_path)
+        try:
+            zi = zip_fi(z, adjusted_cover_path, 2 << 24)
+            return z.open(zi)
+        except KeyError:
+            t = "epub: cover specified in package document, but doesn't exist: %s"
+            log(t % (adjusted_cover_path,))
 
 
 def _get_cover_from_epub2(
@@ -432,9 +494,8 @@ def _get_cover_from_epub2(
 ) -> Optional[str]:
     # <meta name="cover" content="id-to-cover-image"> in <metadata>, then
     # <item> in <manifest>
-    cover_id = package_root.find("./metadata/meta[@name='cover']", package_ns).get(
-        "content"
-    )
+    xn = package_root.find("./metadata/meta[@name='cover']", package_ns)
+    cover_id = xn.get("content") if xn is not None else None
 
     if not cover_id:
         return None
@@ -457,6 +518,8 @@ class MTag(object):
             "ffprobe" if args.no_mutagen or (HAVE_FFPROBE and EXE) else "mutagen"
         )
         self.can_ffprobe = HAVE_FFPROBE and not args.no_mtag_ff
+        self.read_xattrs = args.have_db_xattr
+        self.get = self._get_xattr if self.read_xattrs else self._get_main
         mappings = args.mtm
         or_ffprobe = " or FFprobe"
 
@@ -478,7 +541,13 @@ class MTag(object):
                 msg = "found FFprobe but it was disabled by --no-mtag-ff"
                 self.log(msg, c=3)
 
+        if self.read_xattrs and not self.usable:
+            t = "don't have the necessary dependencies to read conventional media tags, but will read xattrs"
+            self.log(t)
+            self.usable = True
+
         if not self.usable:
+            self._get = None
             if EXE:
                 t = "copyparty.exe cannot use mutagen; need ffprobe.exe to read media tags: "
                 self.log(t + FFMPEG_URL)
@@ -510,13 +579,12 @@ class MTag(object):
                 "album-artist",
                 "tpe2",
                 "aart",
-                "conductor",
                 "organization",
                 "band",
             ],
             ".tn": ["tracknumber", "trck", "trkn", "track"],
             "genre": ["genre", "tcon", "\u00a9gen"],
-            "date": [
+            "tdate": [
                 "original-release-date",
                 "release-date",
                 "date",
@@ -638,12 +706,44 @@ class MTag(object):
 
         return r1
 
-    def get(self, abspath: str) -> dict[str, Union[str, float]]:
+    def _get_xattr(
+        self, abspath: str, vf: dict[str, Any]
+    ) -> dict[str, Union[str, float]]:
+        ret = self._get_main(abspath, vf) if self._get else {}
+        if "db_xattr_no" in vf:
+            try:
+                neg = vf["db_xattr_no"]
+                zsl = os.listxattr(abspath)
+                zsl = [x for x in zsl if x not in neg]
+                for xattr in zsl:
+                    zb = os.getxattr(abspath, xattr)
+                    ret[xattr] = zb.decode("utf-8", "replace")
+            except:
+                self.log("failed to read xattrs from [%s]\n%s" % (abspath, min_ex()), 3)
+        elif "db_xattr_yes" in vf:
+            for xattr in vf["db_xattr_yes"]:
+                if "=" in xattr:
+                    xattr, name = xattr.split("=", 1)
+                else:
+                    name = xattr
+                try:
+                    zs = os.getxattr(abspath, xattr)
+                    ret[name] = zs.decode("utf-8", "replace")
+                except:
+                    pass
+        return ret
+
+    def _get_main(
+        self, abspath: str, vf: dict[str, Any]
+    ) -> dict[str, Union[str, float]]:
         ext = abspath.split(".")[-1].lower()
         if ext not in self.args.au_unpk:
             return self._get(abspath)
 
         ap = au_unpk(self.log, self.args.au_unpk, abspath)
+        if not ap:
+            return {}
+
         ret = self._get(ap)
         if ap != abspath:
             wunlink(self.log, ap, VF_CAREFUL)
@@ -749,6 +849,9 @@ class MTag(object):
             ap = abspath
 
         ret: dict[str, Any] = {}
+        if not ap:
+            return ret
+
         for tagname, parser in sorted(parsers.items(), key=lambda x: (x[1].pri, x[0])):
             try:
                 cmd = [parser.bin, ap]

copyparty/multicast.py

@@ -1,5 +1,5 @@
 # coding: utf-8
-from __future__ import print_function, unicode_literals
+from __future__ import division, print_function, unicode_literals
 
 import socket
 import time
@@ -15,7 +15,7 @@ from ipaddress import (
 )
 
 from .__init__ import MACOS, TYPE_CHECKING
-from .util import Daemon, Netdev, find_prefix, min_ex, spack
+from .util import IP6_LL, IP64_LL, Daemon, Netdev, find_prefix, min_ex, spack
 
 if TYPE_CHECKING:
     from .svchub import SvcHub
@@ -96,7 +96,10 @@ class MCast(object):
     def create_servers(self) -> list[str]:
         bound: list[str] = []
         netdevs = self.hub.tcpsrv.netdevs
-        ips = [x[0] for x in self.hub.tcpsrv.bound]
+        blist = self.hub.tcpsrv.bound
+        if self.args.http_no_tcp:
+            blist = self.hub.tcpsrv.seen_eps
+        ips = [x[0] for x in blist]
 
         if "::" in ips:
             ips = [x for x in ips if x != "::"] + list(
@@ -145,7 +148,7 @@ class MCast(object):
         all_selected = ips[:]
 
         # discard non-linklocal ipv6
-        ips = [x for x in ips if ":" not in x or x.startswith("fe80")]
+        ips = [x for x in ips if ":" not in x or x.startswith(IP6_LL)]
 
         if not ips:
             raise NoIPs()
@@ -183,7 +186,7 @@ class MCast(object):
                 srv.ips[oth_ip.split("/")[0]] = ipaddress.ip_network(oth_ip, False)
 
             # gvfs breaks if a linklocal ip appears in a dns reply
-            ll = {k: v for k, v in srv.ips.items() if k.startswith(("169.254", "fe80"))}
+            ll = {k: v for k, v in srv.ips.items() if k.startswith(IP64_LL)}
             rt = {k: v for k, v in srv.ips.items() if k not in ll}
 
             if self.args.ll or not rt:

copyparty/pwhash.py

@@ -1,5 +1,5 @@
 # coding: utf-8
-from __future__ import print_function, unicode_literals
+from __future__ import division, print_function, unicode_literals
 
 import argparse
 import base64
@@ -14,8 +14,9 @@ try:
     if os.environ.get("PRTY_NO_ARGON2"):
         raise Exception()
 
-    HAVE_ARGON2 = True
     from argon2 import exceptions as argon2ex
+
+    HAVE_ARGON2 = True
 except:
     HAVE_ARGON2 = False
 
@@ -25,6 +26,7 @@ class PWHash(object):
         self.args = args
 
         zsl = args.ah_alg.split(",")
+        zsl = [x.strip() for x in zsl]
         alg = zsl[0]
         if alg == "none":
             alg = ""

copyparty/qrkode.py

@@ -0,0 +1,112 @@
+# coding: utf-8
+from __future__ import division, print_function, unicode_literals
+
+import os
+
+try:
+    if os.environ.get("PRTY_SYS_ALL") or os.environ.get("PRTY_SYS_QRCG"):
+        raise ImportError()
+    from .stolen.qrcodegen import QrCode
+
+    qrgen = QrCode.encode_binary
+    VENDORED = True
+except ImportError:
+    VENDORED = False
+    from qrcodegen import QrCode
+
+if os.environ.get("PRTY_MODSPEC"):
+    from inspect import getsourcefile
+
+    print("PRTY_MODSPEC: qrcode:", getsourcefile(QrCode))
+
+if True:  # pylint: disable=using-constant-test
+    import typing
+    from typing import Any, Optional, Sequence, Union
+
+
+if not VENDORED:
+
+    def _qrgen(data: Union[bytes, Sequence[int]]) -> "QrCode":
+        ret = None
+        V = QrCode.Ecc
+        for e in [V.HIGH, V.QUARTILE, V.MEDIUM, V.LOW]:
+            qr = QrCode.encode_binary(data, e)
+            qr.size = qr._size
+            qr.modules = qr._modules
+            if not ret or ret.size > qr.size:
+                ret = qr
+        return ret
+
+    qrgen = _qrgen
+
+
+def qr2txt(qr: QrCode, zoom: int = 1, pad: int = 4) -> str:
+    tab = qr.modules
+    sz = qr.size
+    if sz % 2 and zoom == 1:
+        tab.append([False] * sz)
+
+    tab = [[False] * sz] * pad + tab + [[False] * sz] * pad
+    tab = [[False] * pad + x + [False] * pad for x in tab]
+
+    rows: list[str] = []
+    if zoom == 1:
+        for y in range(0, len(tab), 2):
+            row = ""
+            for x in range(len(tab[y])):
+                v = 2 if tab[y][x] else 0
+                v += 1 if tab[y + 1][x] else 0
+                row += " ▄▀█"[v]
+            rows.append(row)
+    else:
+        for tr in tab:
+            row = ""
+            for zb in tr:
+                row += " █"[int(zb)] * 2
+            rows.append(row)
+
+    return "\n".join(rows)
+
+
+def qr2png(
+    qr: QrCode,
+    zoom: int,
+    pad: int,
+    bg: Optional[tuple[int, int, int]],
+    fg: Optional[tuple[int, int, int]],
+    ap: str,
+) -> None:
+    from PIL import Image
+
+    tab = qr.modules
+    sz = qr.size
+    psz = sz + pad * 2
+    if bg:
+        img = Image.new("RGB", (psz, psz), bg)
+    else:
+        img = Image.new("RGBA", (psz, psz), (0, 0, 0, 0))
+        fg = (fg[0], fg[1], fg[2], 255)
+    for y in range(sz):
+        for x in range(sz):
+            if tab[y][x]:
+                img.putpixel((x + pad, y + pad), fg)
+    if zoom != 1:
+        img = img.resize((sz * zoom, sz * zoom), Image.Resampling.NEAREST)
+    img.save(ap)
+
+
+def qr2svg(qr: QrCode, border: int) -> str:
+    parts: list[str] = []
+    for y in range(qr.size):
+        sy = border + y
+        for x in range(qr.size):
+            if qr.modules[y][x]:
+                parts.append("M%d,%dh1v1h-1z" % (border + x, sy))
+    t = """\
+<?xml version="1.0" encoding="UTF-8"?>
+<svg xmlns="http://www.w3.org/2000/svg" version="1.1" viewBox="0 0 {0} {0}" stroke="none">
+<rect width="100%" height="100%" fill="#F7F7F7"/>
+<path d="{1}" fill="#111111"/>
+</svg>
+"""
+    return t.format(qr.size + border * 2, " ".join(parts))

copyparty/sftpd.py

@@ -0,0 +1,939 @@
+# coding: utf-8
+from __future__ import division, print_function, unicode_literals
+
+import errno
+import hashlib
+import logging
+import os
+import select
+import socket
+import time
+
+import paramiko
+import paramiko.common
+import paramiko.sftp_attr
+from paramiko.common import AUTH_FAILED, AUTH_SUCCESSFUL
+from paramiko.sftp import (
+    SFTP_FAILURE,
+    SFTP_NO_SUCH_FILE,
+    SFTP_OK,
+    SFTP_OP_UNSUPPORTED,
+    SFTP_PERMISSION_DENIED,
+)
+from queue import Queue
+
+from .__init__ import ANYWIN, TYPE_CHECKING
+from .authsrv import LEELOO_DALLAS, VFS, AuthSrv
+from .bos import bos
+from .util import (
+    VF_CAREFUL,
+    Daemon,
+    ODict,
+    Pebkac,
+    exclude_dotfiles,
+    exclude_dotfiles_ls,
+    exclude_dothidden,
+    exclude_dothidden_ls,
+    ipnorm,
+    min_ex,
+    read_utf8,
+    runhook,
+    sanitize_fn,
+    ub64enc,
+    undot,
+    vjoin,
+    wunlink,
+)
+
+if TYPE_CHECKING:
+    from .svchub import SvcHub
+
+if True:  # pylint: disable=using-constant-test
+    from typing import Any, BinaryIO, Optional, Union
+
+SATTR = paramiko.sftp_attr.SFTPAttributes
+
+
+try:
+    from paramiko.rsakey import serialization
+
+    _load_der_pk = serialization.load_der_private_key
+
+    def fastloader(*a, **ka):
+        ka0 = ka.copy()
+        try:
+            ka["unsafe_skip_rsa_key_validation"] = True
+            return _load_der_pk(*a, **ka)
+        except:
+            return _load_der_pk(*a, **ka0)
+
+except:
+    pass
+
+
+class SSH_Srv(paramiko.ServerInterface):
+    def __init__(self, hub: "SvcHub", addr: Any):
+        self.hub = hub
+        self.args = args = hub.args
+        self.log_func = hub.log
+        self.uname = "*"
+
+        self.addr = addr
+        self.ip = addr[0]
+        if self.ip.startswith("::ffff:"):
+            self.ip = self.ip[7:]
+
+        zsl = []
+        if args.sftp_anon:
+            zsl.append("none")
+        if args.sftp_key2u:
+            zsl.append("publickey")
+        if args.sftp_pw or args.sftp_anon:
+            zsl.append("password")
+        self._auths = ",".join(zsl)
+
+    def log(self, msg: str, c: Union[int, str] = 0) -> None:
+        self.hub.log("sftp:%s" % (self.ip,), msg, c)
+
+    def get_allowed_auths(self, username: str) -> str:
+        return self._auths
+
+    def get_banner(self) -> tuple[Optional[str], Optional[str]]:
+        if self.args.sftpv:
+            self.log("get_banner")
+        t = self.args.sftp_banner
+        if not t:
+            return (None, None)
+        if t.startswith("@"):
+            t = read_utf8(self.log, t[1:], False)
+        if t and not t.endswith("\n"):
+            t += "\n"
+        return (t, "en-US")
+
+    def check_channel_request(self, kind: str, chanid: int) -> int:
+        if self.args.sftpv:
+            self.log("channel-request: %r, %r" % (kind, chanid))
+        if kind == "session":
+            return paramiko.common.OPEN_SUCCEEDED
+        return paramiko.common.OPEN_FAILED_ADMINISTRATIVELY_PROHIBITED
+
+    def check_auth_none(self, username: str) -> int:
+        try:
+            return self._check_auth_none(username)
+        except:
+            self.log("unhandled exception: %s" % (min_ex(),), 1)
+            return AUTH_FAILED
+
+    def _check_auth_none(self, uname: str) -> int:
+        args = self.args
+        if uname != args.sftp_anon or not uname:
+            return AUTH_FAILED
+
+        ipn = ipnorm(self.ip)
+        bans = self.hub.bans
+        if ipn in bans:
+            rt = bans[ipn] - time.time()
+            if rt < 0:
+                self.log("client unbanned")
+                del bans[ipn]
+            else:
+                self.log("client is banned")
+                return AUTH_FAILED
+
+        self.uname = "*"
+        self.log("auth-none OK: *")
+        return AUTH_SUCCESSFUL
+
+    def check_auth_password(self, username: str, password: str) -> int:
+        try:
+            return self._check_auth_password(username, password)
+        except:
+            self.log("unhandled exception: %s" % (min_ex(),), 1)
+            return AUTH_FAILED
+
+    def _check_auth_password(self, uname: str, pw: str) -> int:
+        args = self.args
+        if args.sftpv:
+            logpw = pw
+            if args.log_badpwd == 0:
+                logpw = ""
+            elif args.log_badpwd == 2:
+                zb = hashlib.sha512(pw.encode("utf-8", "replace")).digest()
+                logpw = "%" + ub64enc(zb[:12]).decode("ascii")
+            self.log("auth-pw: %r, %r" % (uname, logpw))
+
+        ipn = ipnorm(self.ip)
+        bans = self.hub.bans
+        if ipn in bans:
+            rt = bans[ipn] - time.time()
+            if rt < 0:
+                self.log("client unbanned")
+                del bans[ipn]
+            else:
+                self.log("client is banned")
+                return AUTH_FAILED
+
+        anon = args.sftp_anon
+        if anon and uname == anon:
+            self.uname = "*"
+            self.log("auth-pw OK: *")
+            return AUTH_SUCCESSFUL
+
+        if not args.sftp_pw:
+            return AUTH_FAILED
+
+        if args.usernames:
+            alts = ["%s:%s" % (uname, pw)]
+        else:
+            alts = [pw, uname]
+
+        attempt = "%s:%s" % (uname, pw)
+        uname = ""
+        asrv = self.hub.asrv
+        for zs in alts:
+            zs = asrv.iacct.get(asrv.ah.hash(zs), "")
+            if zs:
+                uname = zs
+                break
+
+        if args.ipu and uname == "*":
+            uname = args.ipu_iu[args.ipu_nm.map(self.ip)]
+        if args.ipr and uname in args.ipr_u:
+            if not args.ipr_u[uname].map(self.ip):
+                logging.warning("username [%s] rejected by --ipr", uname)
+                return AUTH_FAILED
+
+        if not uname or not (asrv.vfs.aread.get(uname) or asrv.vfs.awrite.get(uname)):
+            g = self.hub.gpwd
+            if g.lim:
+                bonk, ip = g.bonk(self.ip, attempt)
+                if bonk:
+                    logging.warning("client banned: invalid passwords")
+                    bans[self.ip] = bonk
+                    try:
+                        # only possible if multiprocessing disabled
+                        self.hub.broker.httpsrv.bans[ip] = bonk  # type: ignore
+                        self.hub.broker.httpsrv.nban += 1  # type: ignore
+                    except:
+                        pass
+            return AUTH_FAILED
+
+        self.uname = uname
+        self.log("auth-pw OK: %s" % (uname,))
+        return AUTH_SUCCESSFUL
+
+    def check_auth_publickey(self, username: str, key: paramiko.PKey) -> int:
+        try:
+            return self._check_auth_publickey(username, key)
+        except:
+            self.log("unhandled exception: %s" % (min_ex(),), 1)
+            return AUTH_FAILED
+
+    def _check_auth_publickey(self, uname: str, key: paramiko.PKey) -> int:
+        args = self.args
+        if args.sftpv:
+            zs = key.get_name() + "," + key.get_base64()[:32]
+            self.log("auth-key: %r, %r" % (uname, zs))
+
+        ipn = ipnorm(self.ip)
+        bans = self.hub.bans
+        if ipn in bans:
+            rt = bans[ipn] - time.time()
+            if rt < 0:
+                self.log("client unbanned")
+                del bans[ipn]
+            else:
+                self.log("client is banned")
+                return AUTH_FAILED
+
+        anon = args.sftp_anon
+        if anon and uname == anon:
+            self.uname = "*"
+            self.log("auth-key OK: *")
+            return AUTH_SUCCESSFUL
+
+        attempt = "%s %s" % (key.get_name(), key.get_base64())
+        ok = args.sftp_key2u.get(attempt) == uname
+
+        if ok and args.ipr and uname in args.ipr_u:
+            if not args.ipr_u[uname].map(self.ip):
+                logging.warning("username [%s] rejected by --ipr", uname)
+                return AUTH_FAILED
+
+        asrv = self.hub.asrv
+        if not ok or not (asrv.vfs.aread.get(uname) or asrv.vfs.awrite.get(uname)):
+            self.log("auth-key REJECTED: %s" % (uname,))
+            return AUTH_FAILED
+
+        self.uname = uname
+        self.log("auth-key OK: %s" % (uname,))
+        return AUTH_SUCCESSFUL
+
+
+class SFTP_FH(paramiko.SFTPHandle):
+    def __init__(self, flags: int = 0) -> None:
+        self.filename = ""
+        self.readfile: Optional[BinaryIO] = None
+        self.writefile: Optional[BinaryIO] = None
+        super(SFTP_FH, self).__init__(flags)
+
+    def stat(self):
+        try:
+            f = self.readfile or self.writefile
+            return SATTR.from_stat(os.fstat(f.fileno()))
+        except OSError as ex:
+            return paramiko.SFTPServer.convert_errno(ex.errno)
+
+    def chattr(self, attr):
+        # python doesn't have equivalents to fchown or fchmod, so we have to
+        # use the stored filename
+        if not self.writefile:
+            return SFTP_PERMISSION_DENIED
+        try:
+            paramiko.SFTPServer.set_file_attr(self.filename, attr)
+            return SFTP_OK
+        except OSError as ex:
+            return paramiko.SFTPServer.convert_errno(ex.errno)
+
+
+class SFTP_Srv(paramiko.SFTPServerInterface):
+    def __init__(self, ssh: paramiko.ServerInterface, *a, **ka):
+        super(SFTP_Srv, self).__init__(ssh, *a, **ka)
+        self.ssh = ssh
+        self.ip: str = ssh.ip  # type: ignore
+        self.hub: "SvcHub" = ssh.hub  # type: ignore
+        self.uname: str = ssh.uname  # type: ignore
+        self.args = self.hub.args
+        self.asrv: "AuthSrv" = self.hub.asrv
+        self.v = self.args.sftpv
+        self.vv = self.args.sftpvv
+
+        if self.uname == LEELOO_DALLAS:
+            raise Exception("send her back")
+
+        cm = 0
+        for zs, zi in (
+            ("s", SATTR.FLAG_SIZE),
+            ("t", SATTR.FLAG_AMTIME),
+            ("p", SATTR.FLAG_PERMISSIONS),
+            ("u", SATTR.FLAG_UIDGID),
+        ):
+            if zs in self.args.sftp_chattr:
+                cm |= zi
+        self.chattr_mask = cm
+
+        self.vols = [
+            vp
+            for vp, vn in self.asrv.vfs.all_vols.items()
+            if self.uname in vn.axs.uread
+            or self.uname in vn.axs.uwrite
+            or self.uname in vn.axs.uget
+        ]
+        self.vis = set()
+        for zs in self.vols:
+            self.vis.add(zs)
+            while zs:
+                zs = zs.rsplit("/", 1)[0] if "/" in zs else ""
+                self.vis.add(zs)
+
+    def log(self, msg: str, c: Union[int, str] = 0) -> None:
+        self.hub.log("sftp:%s" % (self.ip,), msg, c)
+
+    def v2a(
+        self,
+        vpath: str,
+        r: bool = False,
+        w: bool = False,
+        m: bool = False,
+        d: bool = False,
+    ) -> tuple[str, VFS, str]:
+        # because:
+        #  * maybe some clients use \
+        #  * windows filename limitations
+        # need to allow/translate illegal fn,
+        # but require filesystem-correct rd
+        if ANYWIN or ("\\" in vpath and "/" not in vpath):
+            vpath = vpath.replace("\\", "/")
+        vpath = vpath.strip("/\\")
+        if ANYWIN:
+            rd, fn = os.path.split(vpath)
+            fn = sanitize_fn(fn, "win")
+            vpath = vjoin(rd, fn)
+        # end of windows jank
+
+        vn, rem = self.hub.asrv.vfs.get(vpath, self.uname, r, w, m, d)
+        vpath = vjoin(vn.vpath, rem)
+        rd, fn = os.path.split(vpath)
+        if (
+            w
+            and fn.lower() in vn.flags["emb_all"]
+            and self.uname not in vn.axs.uread
+            and "wo_up_readme" not in vn.flags
+        ):
+            fn = "_wo_" + fn
+            vpath = vjoin(rd, fn)
+            vn, rem = self.hub.asrv.vfs.get(vpath, self.uname, r, w, m, d)
+
+        if not vn.realpath:
+            # return "", vn, rem
+            raise OSError(errno.ENOENT, "no filesystem mounted at [/%s]" % (vpath,))
+
+        ap = vn.canonical(rem, False)
+
+        if "bcasechk" in vn.flags and not vn.casechk(rem, True):
+            raise OSError(errno.ENOENT, "file does not exist case-sensitively")
+
+        return ap, vn, rem
+
+    def list_folder(self, path: str) -> list[SATTR] | int:
+        try:
+            return self._list_folder(path)
+        except Pebkac as ex:
+            if ex.code == 404:
+                self.log("folder 404: %s" % (path,))
+                return SFTP_NO_SUCH_FILE
+            return SFTP_PERMISSION_DENIED
+        except:
+            self.log("unhandled exception: %s" % (min_ex(),), 1)
+            return SFTP_FAILURE
+
+    def _list_folder(self, path: str) -> list[SATTR] | int:
+        if self.v:
+            self.log("ls(%s):" % (path,))
+        path = path.strip("/")
+        try:
+            ap, vn, rem = self.v2a(path, r=True)
+        except Pebkac:
+            try:
+                self.v2a(path, w=True)
+                self.log("ls(%s): [] (write-only)" % (path,))
+                return []  # display write-only folders as empty
+            except:
+                pass
+            if path not in self.vis:
+                self.log("ls(%s): EPERM" % (path,))
+                return SFTP_PERMISSION_DENIED
+            # list of accessible volumes
+            ret = []
+            zi = int(time.time())
+            vst = os.stat_result((16877, -1, -1, 1, 1000, 1000, 8, zi, zi, zi))
+            prefix = path + "/"
+            for vn in self.asrv.vfs.all_nodes.values():
+                if path and not vn.vpath.startswith(prefix):
+                    continue  # vn is parent
+                vname = vn.vpath[len(prefix) :]
+                if "/" in vname or not vname:
+                    continue  # only include vols at current level
+                ret.append(SATTR.from_stat(vst, filename=vn.vpath))
+            ret.sort(key=lambda x: x.filename)
+            self.log("ls(%s): vfs-vols; |%d|" % (path, len(ret)))
+            return ret
+
+        fsroot, vfs_ls, vfs_virt = vn.ls(
+            rem,
+            self.uname,
+            not self.args.no_scandir,
+            [[True, False], [False, True]],
+            throw=True,
+        )
+        vnames = list(vfs_virt)
+        if self.uname not in vn.axs.udot:
+            if "dothidden" in vn.flags and ".hidden" in [x[0] for x in vfs_ls]:
+                vfs_ls = exclude_dothidden_ls(vfs_ls, fsroot)
+                vnames = exclude_dothidden(vnames, fsroot)
+            else:
+                vfs_ls = exclude_dotfiles_ls(vfs_ls)
+                vnames = exclude_dotfiles(vnames)
+        ret = [SATTR.from_stat(x[1], filename=x[0]) for x in vfs_ls]
+        for zs, vn2 in vfs_virt.items():
+            if not vn2.realpath or zs not in vnames:
+                continue
+            st = bos.stat(vn2.realpath)
+            ret.append(SATTR.from_stat(st, filename=zs))
+        ret.sort(key=lambda x: x.filename)
+        self.log("ls(%s): |%d|" % (path, len(ret)))
+        return ret
+
+    def stat(self, path: str) -> SATTR | int:
+        try:
+            return self._stat(path)
+        except:
+            self.log("unhandled exception: %s" % (min_ex(),), 1)
+            return SFTP_FAILURE
+
+    def lstat(self, path: str) -> SATTR | int:
+        try:
+            return self._stat(path)
+        except:
+            self.log("unhandled exception: %s" % (min_ex(),), 1)
+            return SFTP_FAILURE
+
+    def _stat(self, vp: str) -> SATTR | int:
+        vp = vp.strip("/")
+        try:
+            ap, vn, _ = self.v2a(vp)
+            if (
+                self.uname not in vn.axs.uread
+                and self.uname not in vn.axs.uwrite
+                and self.uname not in vn.axs.uget
+            ):
+                self.log("stat(%s): EPERM" % (vp,))
+                return SFTP_PERMISSION_DENIED
+            st = bos.stat(ap)
+            self.log("stat(%s): %s" % (vp, st))
+        except:
+            if vp not in self.vis:
+                self.log("stat(%s): ENOENT" % (vp,))
+                return SFTP_NO_SUCH_FILE
+            zi = int(time.time())
+            st = os.stat_result((16877, -1, -1, 1, 1000, 1000, 8, zi, zi, zi))
+            self.log("stat(%s): vfs-vols")
+        return SATTR.from_stat(st)
+
+    def open(self, path: str, flags: int, attr: SATTR) -> paramiko.SFTPHandle | int:
+        try:
+            return self._open(path, flags, attr)
+        except:
+            self.log("unhandled exception: %s" % (min_ex(),), 1)
+            return SFTP_FAILURE
+
+    def _open(self, vp: str, iflag: int, attr: SATTR) -> paramiko.SFTPHandle | int:
+        if ANYWIN:
+            iflag |= os.O_BINARY
+        if iflag & os.O_WRONLY:
+            rd = False
+            wr = True
+            if iflag & os.O_APPEND:
+                smode = "ab"
+            else:
+                smode = "wb"
+        elif iflag & os.O_RDWR:
+            rd = wr = True
+            if iflag & os.O_APPEND:
+                smode = "a+b"
+            else:
+                smode = "r+b"
+        else:
+            rd = True
+            wr = False
+            smode = "rb"
+
+        try:
+            ap, vn, rem = self.v2a(vp, rd, wr)
+            vf = vn.flags
+        except Pebkac as ex:
+            t = "denied open file [%s], iflag=%s, read=%s, write=%s: %s"
+            self.log(t % (vp, iflag, rd, wr, ex))
+            return SFTP_PERMISSION_DENIED
+
+        self.log("open(%s, %x, %s)" % (vp, iflag, smode))
+
+        if wr:
+            try:
+                st = bos.stat(ap)
+                td = time.time() - st.st_mtime
+                need_unlink = True
+            except:
+                need_unlink = False
+                td = 0
+
+            xbu = vn.flags.get("xbu")
+            if xbu:
+                hr = runhook(
+                    self.log,
+                    None,
+                    self.hub.up2k,
+                    "xbu.sftp",
+                    xbu,
+                    ap,
+                    vp,
+                    "",
+                    "",
+                    "",
+                    0,
+                    0,
+                    "7.3.8.7",
+                    time.time(),
+                    None,
+                )
+                t = hr.get("rejectmsg") or ""
+                if t or hr.get("rc") != 0:
+                    if not t:
+                        t = "upload blocked by xbu server config: %r" % (vp,)
+                    self.log(t, 3)
+                    return SFTP_PERMISSION_DENIED
+
+            self.log("writing to [%s] => [%s]" % (vp, ap))
+
+        if wr and need_unlink:  # type: ignore  # !rm
+            assert td  # type: ignore  # !rm
+            if td >= -1 and td <= self.args.ftp_wt:
+                # within permitted timeframe; allow overwrite or resume
+                do_it = True
+            elif self.args.no_del or self.args.ftp_no_ow:
+                # file too old, or overwrite not allowed; reject
+                do_it = False
+            else:
+                # allow overwrite if user has delete permission
+                do_it = self.uname in vn.axs.udel
+
+            if not do_it:
+                t = "file already exists and no permission to overwrite: %s"
+                self.log(t % (vp,))
+                return SFTP_PERMISSION_DENIED
+
+            # Don't unlink file for append mode
+            elif "a" not in smode:
+                wunlink(self.log, ap, VF_CAREFUL)
+
+        chmod = getattr(attr, "st_mode", None)
+        if chmod is None:
+            chmod = vf.get("chmod_f", 0o644)
+            self.log("open(%s, %x): client did not chmod" % (vp, iflag))
+        else:
+            self.log("open(%s, %x): client set chmod 0%o" % (vp, iflag, chmod))
+
+        try:
+            fd = os.open(ap, iflag, chmod)
+        except OSError as ex:
+            t = "failed to os.open [%s] -> [%s] with iflag [%s] and chmod [%s]: %r"
+            self.log(t % (vp, ap, iflag, chmod, ex), 3)
+            return paramiko.SFTPServer.convert_errno(ex.errno)
+
+        if iflag & os.O_CREAT:
+            paramiko.SFTPServer.set_file_attr(ap, attr)
+
+        try:
+            f = os.fdopen(fd, smode)
+        except OSError as ex:
+            t = "failed to os.fdpen [%s] -> [%s] with smode [%s]: %r"
+            self.log(t % (vp, ap, smode, ex), 3)
+            return paramiko.SFTPServer.convert_errno(ex.errno)
+
+        ret = SFTP_FH(iflag)
+        ret.filename = ap
+        ret.readfile = f if rd else None
+        ret.writefile = f if wr else None
+        return ret
+
+    def remove(self, path: str) -> int:
+        try:
+            return self._remove(path)
+        except:
+            self.log("unhandled exception: %s" % (min_ex(),), 1)
+            return SFTP_FAILURE
+
+    def _remove(self, vp: str) -> int:
+        self.log("rm(%s)" % (vp,))
+        if self.args.no_del:
+            self.log("The delete feature is disabled in server config")
+            return SFTP_PERMISSION_DENIED
+        try:
+            self.hub.up2k.handle_rm(self.uname, self.ip, [vp], [], False, False)
+            self.log("rm(%s): ok" % (vp,))
+            return SFTP_OK
+        except Pebkac as ex:
+            t = "denied delete [%s]: %s"
+            self.log(t % (vp, ex))
+            if str(ex).startswith("file not found"):
+                return SFTP_NO_SUCH_FILE
+            try:
+                # write-only client trying to rm before upload?
+                ap, vn, _ = self.v2a(vp)
+                if (
+                    self.uname not in vn.axs.uread
+                    and self.uname not in vn.axs.uwrite
+                    and self.uname not in vn.axs.uget
+                ):
+                    self.log("rm(%s): EPERM" % (vp,))
+                    return SFTP_PERMISSION_DENIED
+                if not bos.path.exists(ap):
+                    self.log(" `- file didn't exist; returning ENOENT")
+                    return SFTP_NO_SUCH_FILE
+            except:
+                pass
+            return SFTP_PERMISSION_DENIED
+        except OSError as ex:
+            self.log("failed: rm(%s): %r" % (vp, ex))
+            return paramiko.SFTPServer.convert_errno(ex.errno)
+
+    def rename(self, oldpath: str, newpath: str) -> int:
+        try:
+            return self._rename(oldpath, newpath)
+        except:
+            self.log("unhandled exception: %s" % (min_ex(),), 1)
+            return SFTP_FAILURE
+
+    def _rename(self, svp: str, dvp: str) -> int:
+        self.log("mv(%s, %s)" % (svp, dvp))
+        if self.args.no_mv:
+            self.log("The rename/move feature is disabled in server config")
+        svp = svp.strip("/")
+        dvp = dvp.strip("/")
+        try:
+            self.hub.up2k.handle_mv("", self.uname, self.ip, svp, dvp)
+            return SFTP_OK
+        except Pebkac as ex:
+            t = "denied rename [%s] to [%s]: %s"
+            self.log(t % (svp, dvp, ex))
+            return SFTP_PERMISSION_DENIED
+        except OSError as ex:
+            self.log("mv(%s, %s): %r" % (svp, dvp, ex))
+            return paramiko.SFTPServer.convert_errno(ex.errno)
+
+    def mkdir(self, path: str, attr: SATTR) -> int:
+        try:
+            return self._mkdir(path, attr)
+        except:
+            self.log("unhandled exception: %s" % (min_ex(),), 1)
+            return SFTP_FAILURE
+
+    def _mkdir(self, vp: str, attr: SATTR) -> int:
+        self.log("mkdir(%s)" % (vp,))
+        try:
+            ap, vn, _ = self.v2a(vp, w=True)
+            bos.makedirs(ap, vf=vn.flags)  # filezilla expects this
+            if attr is not None:
+                paramiko.SFTPServer.set_file_attr(ap, attr)
+            return SFTP_OK
+        except Pebkac as ex:
+            t = "denied mkdir [%s]: %s"
+            self.log(t % (vp, ex))
+            return SFTP_PERMISSION_DENIED
+        except OSError as ex:
+            self.log("mkdir(%s): %r" % (vp, ex))
+            return paramiko.SFTPServer.convert_errno(ex.errno)
+
+    def rmdir(self, path: str) -> int:
+        try:
+            return self._rmdir(path)
+        except:
+            self.log("unhandled exception: %s" % (min_ex(),), 1)
+            return SFTP_FAILURE
+
+    def _rmdir(self, vp: str) -> int:
+        self.log("rmdir(%s)" % (vp,))
+        try:
+            ap, _, _ = self.v2a(vp, d=True)
+            bos.rmdir(ap)
+            return SFTP_OK
+        except Pebkac as ex:
+            t = "denied rmdir [%s]: %s"
+            self.log(t % (vp, ex))
+            return SFTP_PERMISSION_DENIED
+        except OSError as ex:
+            self.log("rmdir(%s): %r" % (vp, ex))
+            return paramiko.SFTPServer.convert_errno(ex.errno)
+
+    def chattr(self, path: str, attr: SATTR) -> int:
+        try:
+            return self._chattr(path, attr)
+        except:
+            self.log("unhandled exception: %s" % (min_ex(),), 1)
+            return SFTP_FAILURE
+
+    def _chattr(self, vp: str, attr: SATTR) -> int:
+        self.log("chattr(%s, %s)" % (vp, attr))
+        try:
+            attr._flags &= self.chattr_mask
+            ap, _, _ = self.v2a(vp, w=True, d=True)
+            paramiko.SFTPServer.set_file_attr(ap, attr)
+            return SFTP_OK
+        except Pebkac as ex:
+            t = "denied chattr [%s]: %s"
+            self.log(t % (vp, ex))
+            return SFTP_PERMISSION_DENIED
+        except OSError as ex:
+            self.log("chattr(%s): %r" % (vp, ex))
+            return paramiko.SFTPServer.convert_errno(ex.errno)
+
+    def symlink(self, target_path: str, path: str) -> int:
+        return SFTP_OP_UNSUPPORTED
+
+    def readlink(self, path: str) -> str | int:
+        return path
+
+    def canonicalize(self, path: str) -> str:
+        return "/%s" % (undot(path),)
+
+
+class Transport2(paramiko.Transport):
+    def __init__(self, cli, addr):
+        self.addr = addr
+        super(Transport2, self).__init__(cli)
+
+    def _check_banner(self):
+        try:
+            return super()._check_banner()
+        except Exception as ex:
+            if "Error reading SSH protocol bann" in str(ex):
+                self.logger.warning("%s rejected: %s" % (self.addr, ex))
+                self.addr = None
+            else:
+                raise
+
+
+class Sftpd(object):
+    def __init__(self, hub: "SvcHub") -> None:
+        self.hub = hub
+        self.args = args = hub.args
+        self.log_func = hub.log
+        self.srv: list[socket.socket] = []
+        self.bound: list[str] = []
+        self.hs_q = Queue()
+        self.sessions = {}
+
+        ips = args.sftp_i
+        if "::" in ips:
+            ips.append("0.0.0.0")
+
+        ips = [x for x in ips if not x.startswith(("unix:", "fd:"))]
+
+        if args.sftp4:
+            ips = [x for x in ips if ":" not in x]
+
+        if not ips:
+            self.log("cannot start sftp-server; no compatible IPs in -i", 1)
+            return
+
+        if args.sftp_fastldr:
+            serialization.load_der_private_key = fastloader  # type: ignore
+        else:
+            self.log("loading hostkeys...")
+        self.hostkeys = []
+        hostkeytypes = (
+            ("ed25519", "Ed25519Key", {}),  # best
+            ("ecdsa", "ECDSAKey", {"bits": 384}),
+            ("rsa", "RSAKey", {"bits": 4096}),
+            ("dsa", "DSSKey", {}),  # worst
+        )
+        for fname, aname, opts in hostkeytypes:
+            fpath = "%s/ssh_host_%s_key" % (args.sftp_hostk, fname.lower())
+            try:
+                pkey = getattr(paramiko, aname).from_private_key_file(fpath)
+            except Exception as ex:
+                try:
+                    genfun = getattr(paramiko, aname).generate
+                except Exception as ex2:
+                    if args.sftpv or fname not in ("dsa", "ed25519"):
+                        # dsa dropped in 4.0
+                        # ed25519 not supported yet
+                        self.log("cannot generate %s hostkey: %r" % (aname, ex2), 3)
+                    continue
+                self.log("generating hostkey [%s] due to %r" % (fpath, ex))
+                pkey = genfun(**opts)
+                pkey.write_private_key_file(fpath)
+                pkey = getattr(paramiko, aname).from_private_key_file(fpath)
+            self.hostkeys.append(pkey)
+            if args.sftpv:
+                self.log("loaded hostkey %r" % (pkey,))
+        if args.sftp_fastldr:
+            serialization.load_der_private_key = _load_der_pk  # type: ignore
+
+        ips = list(ODict.fromkeys(ips))  # dedup
+
+        for ip in ips:
+            self._bind(ip)
+
+        self.log("listening @ %s port %s" % (self.bound, args.sftp))
+
+        for n in range(self.args.sftp_hs_n):
+            Daemon(self.acceptor, "sftp-hs-%d" % (n,))
+
+    def log(self, msg: str, c: Union[int, str] = 0) -> None:
+        self.hub.log("sftp", msg, c)
+
+    def _bind(self, ip: str) -> None:
+        port = self.args.sftp
+        try:
+            ipv = socket.AF_INET6 if ":" in ip else socket.AF_INET
+            srv = socket.socket(ipv, socket.SOCK_STREAM)
+            if not ANYWIN or self.args.reuseaddr:
+                srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+            srv.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+            srv.settimeout(0)  # == srv.setblocking(False)
+            try:
+                srv.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, False)
+            except:
+                pass  # will create another ipv4 socket instead
+            if getattr(self.args, "freebind", False):
+                srv.setsockopt(socket.SOL_IP, socket.IP_FREEBIND, 1)
+            srv.bind((ip, port))
+            srv.listen(10)
+            self.srv.append(srv)
+            self.bound.append(ip)
+        except Exception as ex:
+            if ip == "0.0.0.0" and "::" in self.bound:
+                try:
+                    srv.close()  # type: ignore
+                except:
+                    pass
+                return  # dualstack
+            self.log("could not listen on (%s,%s): %r" % (ip, port, ex), 3)
+
+    def _accept(self, srv: socket.socket) -> None:
+        cli, addr = srv.accept()
+        # cli.settimeout(0)  # == srv.setblocking(False)
+        self.log("%r is connecting" % (addr,))
+        self.hs_q.put((cli, addr))
+
+    def acceptor(self) -> None:
+        while True:
+            try:
+                cli, addr = self.hs_q.get()
+                self._accept2(cli, addr)
+            except:
+                time.sleep(1)
+
+    def _accept2(self, cli, addr) -> None:
+        tra = Transport2(cli, addr)
+        for hkey in self.hostkeys:
+            tra.add_server_key(hkey)
+        tra.set_subsystem_handler("sftp", paramiko.SFTPServer, SFTP_Srv)
+        psrv = SSH_Srv(self.hub, addr)
+        try:
+            tra.start_server(server=psrv)
+        except Exception as ex:
+            if tra.addr:
+                self.log("%r could not establish connection: %r" % (addr, ex), 3)
+            cli.close()
+            return
+
+        chan = tra.accept(timeout=self.args.sftp_hs_t)
+        if chan is None:
+            self.log("%r did not open an sftp channel" % (addr,), 3)
+            cli.close()
+            return
+
+        self.sessions[addr] = (chan, tra, psrv)
+        # tra.join()
+        # self.log("%r disconnected" % (addr,))
+
+    def run(self):
+        lgr = logging.getLogger("paramiko.transport")
+        lgr.setLevel(logging.DEBUG if self.args.sftpvv else logging.INFO)
+
+        if self.args.no_poll:
+            fun = self._run_select
+        else:
+            fun = self._run_poll
+        Daemon(fun, "sftpd")
+
+    def _run_select(self):
+        while not self.hub.stopping:
+            rx, _, _ = select.select(self.srv, [], [], 180)
+            for sck in rx:
+                self._accept(sck)
+
+    def _run_poll(self):
+        fd2sck = {}
+        poll = select.poll()
+        for sck in self.srv:
+            fd = sck.fileno()
+            fd2sck[fd] = sck
+            poll.register(fd, select.POLLIN)
+        while not self.hub.stopping:
+            pr = poll.poll(180 * 1000)
+            rx = [fd2sck[x[0]] for x in pr if x[1] & select.POLLIN]
+            for sck in rx:
+                self._accept(sck)

copyparty/smbd.py

@@ -12,7 +12,7 @@ from types import SimpleNamespace
 from .__init__ import ANYWIN, EXE, TYPE_CHECKING
 from .authsrv import LEELOO_DALLAS, VFS
 from .bos import bos
-from .util import Daemon, absreal, min_ex, pybin, runhook, vjoin
+from .util import Daemon, absreal, min_ex, pybin, runhook
 
 if True:  # pylint: disable=using-constant-test
     from typing import Any, Union
@@ -89,13 +89,15 @@ class SMB(object):
         smbserver.isInFileJail = self._is_in_file_jail
         self._disarm()
 
-        ip = next((x for x in self.args.i if ":" not in x), None)
+        zs = " " if self.args.smb6 else ":"
+        ip = next((x for x in self.args.smb_i if zs not in x), None)
         if not ip:
-            self.log("smb", "IPv6 not supported for SMB; listening on 0.0.0.0", 3)
+            self.log("smb", "IPv6 not enabled with --smb6; listening on 0.0.0.0", 3)
             ip = "0.0.0.0"
 
         port = int(self.args.smb_port)
-        srv = smbserver.SimpleSMBServer(listenAddress=ip, listenPort=port)
+        kw = {"ipv6": True} if ":" in ip else {}
+        srv = smbserver.SimpleSMBServer(listenAddress=ip, listenPort=port, **kw)
         try:
             if self.accs:
                 srv.setAuthCallback(self._auth_cb)
@@ -121,6 +123,7 @@ class SMB(object):
 
         self.srv = srv
         self.stop = srv.stop
+        ip = "[%s]" % (ip,) if kw else ip
         self.log("smb", "listening @ {}:{}".format(ip, port))
 
     def nlog(self, msg: str, c: Union[int, str] = 0) -> None:
@@ -191,7 +194,7 @@ class SMB(object):
         vfs, rem = self.asrv.vfs.get(vpath, uname, *perms)
         if not vfs.realpath:
             raise Exception("unmapped vfs")
-        return vfs, vjoin(vfs.realpath, rem)
+        return vfs, vfs.canonical(rem, False)
 
     def _listdir(self, vpath: str, *a: Any, **ka: Any) -> list[str]:
         vpath = vpath.replace("\\", "/").lstrip("/")
@@ -246,24 +249,29 @@ class SMB(object):
 
             ap = absreal(ap)
             xbu = vfs.flags.get("xbu")
-            if xbu and not runhook(
-                self.nlog,
-                None,
-                self.hub.up2k,
-                "xbu.smb",
-                xbu,
-                ap,
-                vpath,
-                "",
-                "",
-                "",
-                0,
-                0,
-                "1.7.6.2",
-                time.time(),
-                "",
-            ):
-                yeet("blocked by xbu server config: %r" % (vpath,))
+            if xbu:
+                hr = runhook(
+                    self.nlog,
+                    None,
+                    self.hub.up2k,
+                    "xbu.smb",
+                    xbu,
+                    ap,
+                    vpath,
+                    "",
+                    "",
+                    "",
+                    0,
+                    0,
+                    "1.7.6.2",
+                    time.time(),
+                    None,
+                )
+                t = hr.get("rejectmsg") or ""
+                if t or hr.get("rc") != 0:
+                    if not t:
+                        t = "blocked by xbu server config: %r" % (vpath,)
+                    yeet(t)
 
         ret = bos.open(ap, flags, *a, mode=chmod, **ka)
         if wr:
@@ -318,7 +326,7 @@ class SMB(object):
             t = "blocked rename (no-move-acc %s): /%s @%s"
             yeet(t % (vfs1.axs.umove, vp1, uname))
 
-        self.hub.up2k.handle_mv(uname, "1.7.6.2", vp1, vp2)
+        self.hub.up2k.handle_mv("", uname, "1.7.6.2", vp1, vp2)
         try:
             bos.makedirs(ap2, vf=vfs2.flags)
         except:
@@ -373,7 +381,7 @@ class SMB(object):
             t = "blocked utime (no-write-acc %s): /%s @%s"
             yeet(t % (vfs.axs.uwrite, vpath, uname))
 
-        return bos.utime(ap, times)
+        bos.utime_c(info, ap, int(times[1]), False)
 
     def _p_exists(self, vpath: str) -> bool:
         # ap = "?"