cmd/env.rc

   1 CURL="${CURL:-curl}"
   2 ZSTD="${ZSTD:-zstdmt -19}"
   3 WGET="${WGET:-wget}"
   4 PARALLEL="${PARALLEL:-parallel --bar --shuf}"
   5
   6 FEEDER_USER_AGENT="${FEEDER_USER_AGENT:-stargrave.org-feeder/0.1.0}"
   7 #FEEDER_CURL_VERBOSE=1
   8 FEEDER_CURL_MAX_TIME=${FEEDER_CURL_MAX_TIME:-20}
   9 FEEDER_MAX_ITEMS=${FEEDER_MAX_ITEMS:-100}
  10 FEEDER_DOWNLOAD_JOBS=${FEEDER_DOWNLOAD_JOBS:-10}
  11 FEEDER_PARSE_JOBS=${FEEDER_PARSE_JOBS:-0}
  12
  13 command -v sha512 >/dev/null && SHA512="sha512" || SHA512="sha512sum --binary"
  14
  15 #MAILCAPS="${MAILCAPS:-$cmds/../contrib/mailcap}"
  16
  17 _feeder_warc_compress() {
  18     $HOME/work/tofuproxy/warc-extract.cmd -for-enzstd "$1" |
  19     $HOME/work/tofuproxy/cmd/zstd/enzstd > "$1".zst
  20     rm "$1"
  21 }
  22
  23 FEEDER_WARC_COMPRESS=_feeder_warc_compress
  24 FEEDER_WARC_COMPRESS=: # do not compress
  25 #export http_proxy=http://localhost:8080
  26 #export https_proxy=$http_proxy