2 ZSTD="${ZSTD:-zstdmt -19}"
4 PARALLEL="${PARALLEL:-parallel --bar --shuf}"
6 FEEDER_USER_AGENT="${FEEDER_USER_AGENT:-stargrave.org-feeder/0.1.0}"
8 FEEDER_MAX_ITEMS=${FEEDER_MAX_ITEMS:-100}
9 FEEDER_DOWNLOAD_JOBS=${FEEDER_DOWNLOAD_JOBS:-10}
10 FEEDER_PARSE_JOBS=${FEEDER_PARSE_JOBS:-0}
12 command -v sha512 >/dev/null && SHA512="sha512" || SHA512="sha512sum --binary"
14 #MAILCAPS="${MAILCAPS:-$cmds/../contrib/mailcap}"
16 _feeder_warc_compress() {
17 $HOME/work/tofuproxy/warc-extract.cmd -for-enzstd "$1" |
18 $HOME/work/tofuproxy/cmd/zstd/enzstd > "$1".zst
22 FEEDER_WARC_COMPRESS=_feeder_warc_compress
23 FEEDER_WARC_COMPRESS=: # do not compress