Various refactoring and parametrizing

[feeder.git] / cmd / warcs.zsh

diff --git a/cmd/download-warcs.zsh b/cmd/warcs.zsh
similarity index 82%
rename from cmd/download-warcs.zsh
rename to cmd/warcs.zsh
index 30d8202f4d536d6aacf6ac5eb0f25e4095ad3c47..4e3efd647f2334de34edc26fe325539d69b0b7f7 100755 (executable)
--- a/cmd/download-warcs.zsh
+++ b/cmd/warcs.zsh
@@ -1,6 +1,8 @@
 #!/usr/bin/env zsh
 set -e
-fpath=($0:h:a/functions.zsh $fpath)
+cmds=$0:h:a
+. $cmds/env.rc
+fpath=($cmds/functions.zsh $fpath)
 dst=$2:a
 cd $1
 [[ -n "$dst" ]] || { dst=warcs ; dst=$dst:a }
@@ -9,6 +11,7 @@ autoload url-to-filename
 zmodload -F zsh/datetime b:strftime
 setopt EXTENDED_GLOB
 wget_opts=(
+    --user-agent="$FEEDER_USER_AGENT"
     --page-requisites
     --compression=auto
     --no-warc-keep-log
@@ -29,7 +32,7 @@ for new (new/*(N)) {
             wget_opts=(--output-document=$tmp $wget_opts)
         }
         fn=$(strftime %Y%m%d-%H%M%S)-$(url-to-filename $url)
-        wget $wget_opts --output-file=warcs.log --warc-file=$dst/$fn $url
+        ${=WGET} $wget_opts --output-file=warcs.log --warc-file=$dst/$fn $url
         print $dst/$fn.warc
     done < $new
 }