--- /dev/null
+#!/bin/sh -e
+
+cd "$1"
+rm -fv download.hash etag feed hdr out
--- /dev/null
+#!/bin/sh -e
+
+PROXY="--proxy http://localhost:8080/"
+cd "$1"
+read url < url
+[ -s etag ] && etag_compare="--etag-compare etag" || etag_compare=""
+[ -r out ] && time_cond="--time-cond out" || time_cond=""
+[ -z "$FEEDER_CURL_VERBOSE" ] && silent="--silent" || silent="--verbose"
+curl --fail \
+ --user-agent "go.stargrave.org-feeder/0.1.0" \
+ --compressed \
+ --location --max-redirs 2 \
+ --dump-header hdr \
+ --output out \
+ --remote-time \
+ --etag-save etag \
+ $PROXY \
+ $etag_compare \
+ $time_cond \
+ $silent \
+ "$url" >&2
+if [ -s out ] ; then
+ zstdmt -19 < out > feed.zst
+ touch -r out feed.zst
+ truncate -s 0 out
+ touch -r feed.zst out
+fi
+sha512 < feed.zst > download.hash
-find feeds -name title | sort | xargs redo-ifchange
+#!/bin/sh -e
+
cat <<EOF
set mail_check_stats
set mail_check_stats_interval=5
mailboxes search
EOF
+
for f in feeds/* ; do
[ -s $f/title ] || {
echo unreadable $f/title >&2
--- /dev/null
+#!/bin/sh -e
+
+cmds="$(dirname "$(realpath "$0")")"
+cd "$1"
+[ -s parse.hash ] && hash_our=`cat parse.hash` || hash_our=""
+[ -s download.hash ] && hash_their=`cat download.hash` || hash_their=""
+[ "$hash_our" != "$hash_their" ] || exit 0
+zstd -d < feed.zst | $cmds/feed2mdir/feed2mdir . > title.tmp
+mv title.tmp title
+echo $hash_their > parse.hash
--- /dev/null
+print-joglog-failed() {
+ local row
+ read row
+ local cols=(${(s: :)row})
+ local exitvalI=${cols[(i)Exitval]}
+ while read row ; do
+ cols=(${(s: :)row})
+ [[ ${cols[$exitvalI]} -ne 0 ]] || continue
+ print "Failed: ${cols[$#cols]}"
+ done
+}
+++ /dev/null
-d=${1%/*}
-cd $d
-rm etag feed* hdr out
+++ /dev/null
-PROXY="--proxy http://localhost:8080/"
-d=${1%/*}
-[ -z "$FEEDER_SKIP_DOWNLOAD" ] || {
- echo temporarily skipping feed download >&2
- sha512 < $d/feed
- exit
-}
-read url < $d/url
-[ -s $d/etag ] && etag_compare="--etag-compare $d/etag" || etag_compare=""
-[ -z "$FEEDER_CURL_VERBOSE" ] && silent="--silent" || silent="--verbose"
-curl --fail \
- --user-agent "go.stargrave.org-feeder/0.1.0" \
- --compressed \
- --location --max-redirs 2 \
- --dump-header $d/hdr \
- --output $d/out \
- --remote-time \
- --time-cond $d/out \
- --etag-save $d/etag \
- $PROXY \
- $silent \
- $etag_compare \
- "$url" >&2
-if [ -s $d/out ] ; then
- cp -a $d/out $d/feed
- truncate -s 0 $d/out
- touch -r $d/feed $d/out
-fi
-sha512 < $d/feed
+++ /dev/null
-d=${1%/*}
-FEEDER_SKIP_DOWNLOAD=1 redo-ifchange $d/feed.download
-cmd/feed2mdir/feed2mdir $d < $d/feed > $d/title
\input texinfo
@documentencoding UTF-8
-@settitle feeder
+@settitle go.stargrave.org/feeder
@copying
Copyright @copyright{} 2022 @email{stargrave@@stargrave.org, Sergey Matveev}
@url{https://en.wikipedia.org/wiki/Maildir, Maildir}.
@item Automatically generated @url{http://www.mutt.org/, Mutt} source
file contains convenient options and human-readable mailboxes list.
-@item @url{http://cr.yp.to/redo.html, redo} is used for parallel
- invocation of download/parsing jobs and skipping parsing of
- unchanged feeds.
+@item @url{https://www.gnu.org/software/parallel/, GNU parallel} for
+ parallel invocation of download/parsing jobs.
@item @url{https://www.djcbsoftware.nl/code/mu/, mu} utilities perform
indexing and searching among the messages. Of course it would be
trivial to use @url{https://www.lesbonscomptes.com/recoll/, recoll}
or @url{https://notmuchmail.org/, notmuch} instead. If you need that.
-@item Bunch of @url{https://www.zsh.org/, Zsh} helper scripts, that are
- completely optional and pretty trivial.
+@item Bunch of optional @url{https://www.zsh.org/, Zsh} helper scripts.
@end itemize
+But of course it has its price: original feed data, its Base64-encoded
+copy in mailbox and another copy in @command{mu}'s Xapian database.
+
@insertcopying
@include storage.texi
@code{mtime} (for @code{If-Modified-Since} header generation),
@code{ETag} and response headers for debugging.
-@item feed
-It contains the content itself.
-
-@item feed.download
-Used as intermediate target for the @command{redo} build system and
-contains SHA-512 hash of the @file{feed} file. It is used to nearly
-completely skip file copying/moving on filesystem if feed was not
-modified.
+@item feed.zst
+It contains the content itself. Compressed with
+@url{https://facebook.github.io/zstd/, Zstandard}.
+
+@item download.hash, parse.hash
+SHA-512 hash of the @file{feed.zst}, used to determine if feed was
+updated and parser has to do the job.
@item title
Automatically generated file with the title of the feed.
@item Download your feed(s) data
-Downloading is implemented in @command{redo}'s
-@file{default.download.do} file. Probably you want to change its default
-@env{$PROXY} value. It uses @command{curl}, that is aware of
-@code{If-Modified-Since} and @code{ETag} headers, compressed content
-encodings and HTTP redirections.
-
-You can invoke feed downloading like that:
-
@example
-$ redo feeds/blog.stargrave.org_russian_feed.atom/feed.download
+$ cmd/download.sh feeds/blog.stargrave.org_russian_feed.atom
+$ ./feeds-download.zsh # to invoke parallel downloading of everything
@end example
+Probably you want to change its default @env{$PROXY} value. It uses
+@command{curl}, that is aware of @code{If-Modified-Since} and
+@code{ETag} headers, compressed content encodings and HTTP redirections.
If you want to see verbose output, then set @env{FEEDER_CURL_VERBOSE=1}.
-As a rule, you wish to run all feeds downloading in parallel. You can
-use @file{feeds-download.zsh}, that just invokes @command{redo-ifchange}
-with @option{-f} option (forceful rebuild). Why that? Because most
-@command{redo} implementations (that forces target building) do not
-parallelize specified targets build. But you can also use @command{parallel}:
-
-@example
-$ parallel "redo @{@}/feed.download" ::: feeds/*
-@end example
-
@item Parse your feeds
-Parsing (and Maildir filling) is implemented in @command{redo}'s
-@file{default.parse.do} file. It calls @command{cmd/feed2mdir/feed2mdir}
-utility, that read @file{feeds/FEED/feed} from stdin, takes Maildir
-directory as its first argument and prints feed's title.
-
-You can use @file{feeds-parse.zsh} helper or invoke @command{parallel}
-as in example above, replacing @code{.download} with @code{.parse}.
-
@example
-$ ./feeds-parse.zsh
+$ cmd/parse.sh feeds/blog.stargrave.org_russian_feed.atom
+$ ./feeds-parse.zsh # to parse all feeds in parallel
@end example
@item Run Mutt
@item If you want to clean download state
@example
-$ redo feeds/FEED/feed.clean
+$ cmd/download-clean.sh feed/FEED
@end example
@end table
+++ /dev/null
-#!/bin/sh -e
-
-redo-ifchange mutt.rc
-mutt -e "source mutt.rc" -y
--- /dev/null
+#!/usr/bin/env zsh
+set -e
+$0:h/cmd/muttrc-gen.sh > mutt.rc
+mutt -e "source mutt.rc" -y
#!/usr/bin/env zsh
-set -e
-redo-ifchange -f -j 10 `for f (feeds/*) print $f/feed.download`
+cmds=$0:h:a/cmd
+parallel --joblog download.log "$cmds/download.sh {}" ::: feeds/*
+. $cmds/print-failed.zsh.rc
+print-joglog-failed < download.log
#!/usr/bin/env zsh
-set -e
-redo-ifchange -j 16 `for f (feeds/*) print $f/feed.parse`
+cmds=$0:h:a/cmd
+parallel --joblog parse.log "$cmds/parse.sh {}" ::: feeds/*
+. $cmds/print-failed.zsh.rc
+print-joglog-failed < parse.log