]> Sergey Matveev's repositories - feeder.git/blob - cmd/download-encs.zsh
Enclosures and WARCs downloader
[feeder.git] / cmd / download-encs.zsh
1 #!/usr/bin/env zsh
2 set -e
3 fpath=($0:h:a/functions.zsh $fpath)
4 dst=$2:a
5 cd $1
6 autoload url-to-filename
7 zmodload -F zsh/datetime b:strftime
8 setopt EXTENDED_GLOB
9 for new (new/*(N)) {
10     while read line ; do
11         [[ "$line" != "" ]] || break
12         cols=(${(s: :)line})
13         [[ $cols[1] = "X-Enclosure:" ]] || continue
14         url=$cols[2]
15         [[ -n "$url" ]]
16         fn=$(url-to-filename $url)-$(strftime %Y%m%d-%H%M%S)
17         wget --output-file=enc.log --output-document=$dst/$fn $url
18         print $fn
19     done < $new
20 }