CURL="${CURL:-curl}" ZSTD="${ZSTD:-zstdmt -19}" WGET="${WGET:-wget}" PARALLEL="${PARALLEL:-parallel --bar --shuf}" FEEDER_USER_AGENT="${FEEDER_USER_AGENT:-stargrave.org-feeder/0.1.0}" #FEEDER_CURL_VERBOSE=1 FEEDER_CURL_MAX_TIME=${FEEDER_CURL_MAX_TIME:-20} FEEDER_MAX_ITEMS=${FEEDER_MAX_ITEMS:-100} FEEDER_DOWNLOAD_JOBS=${FEEDER_DOWNLOAD_JOBS:-10} FEEDER_PARSE_JOBS=${FEEDER_PARSE_JOBS:-0} command -v sha512 >/dev/null && SHA512="sha512" || SHA512="sha512sum --binary" #MAILCAPS="${MAILCAPS:-$cmds/../contrib/mailcap}" _feeder_warc_compress() { $HOME/work/tofuproxy/warc-extract.cmd -for-enzstd "$1" | $HOME/work/tofuproxy/cmd/zstd/enzstd > "$1".zst rm "$1" } FEEDER_WARC_COMPRESS=_feeder_warc_compress FEEDER_WARC_COMPRESS=: # do not compress