command -v sha512 >/dev/null && SHA512="sha512" || SHA512="sha512sum --binary"
#MAILCAPS="${MAILCAPS:-$cmds/../contrib/mailcap}"
+
+_feeder_warc_compress() {
+ $HOME/work/tofuproxy/warc-extract.cmd -for-enzstd "$1" |
+ $HOME/work/tofuproxy/cmd/zstd/enzstd > "$1".zst
+ rm "$1"
+}
+
+FEEDER_WARC_COMPRESS=_feeder_warc_compress
+FEEDER_WARC_COMPRESS=: # do not compress
}
fn=$(strftime %Y%m%d-%H%M%S)-$(url-to-filename $url)
${=WGET} $wget_opts --output-file=warcs.log --warc-file=$dst/$fn $url
- print $dst/$fn.warc
+ $FEEDER_WARC_COMPRESS $dst/$fn.warc
+ print $dst/$fn.warc*
done < $new
}