CURL="${CURL:-curl}"
ZSTD="${ZSTD:-zstdmt -19}"
WGET="${WGET:-wget}"
-PARALLEL="${PARALLEL:-parallel --bar}"
+PARALLEL="${PARALLEL:-parallel --bar --shuf}"
-FEEDER_USER_AGENT="${FEEDER_USER_AGENT:-go.stargrave.org-feeder/0.1.0}"
+FEEDER_USER_AGENT="${FEEDER_USER_AGENT:-stargrave.org-feeder/0.1.0}"
#FEEDER_CURL_VERBOSE=1
+FEEDER_CURL_MAX_TIME=${FEEDER_CURL_MAX_TIME:-20}
FEEDER_MAX_ITEMS=${FEEDER_MAX_ITEMS:-100}
FEEDER_DOWNLOAD_JOBS=${FEEDER_DOWNLOAD_JOBS:-10}
FEEDER_PARSE_JOBS=${FEEDER_PARSE_JOBS:-0}
command -v sha512 >/dev/null && SHA512="sha512" || SHA512="sha512sum --binary"
#MAILCAPS="${MAILCAPS:-$cmds/../contrib/mailcap}"
+
+_feeder_warc_compress() {
+ $HOME/work/tofuproxy/warc-extract.cmd -for-enzstd "$1" |
+ $HOME/work/tofuproxy/cmd/zstd/enzstd > "$1".zst
+ rm "$1"
+}
+
+FEEDER_WARC_COMPRESS=_feeder_warc_compress
+FEEDER_WARC_COMPRESS=: # do not compress
+export http_proxy=http://localhost:8080
+export https_proxy=$http_proxy