Per-feed max number of entries

diff --git a/cmd/feed2mdir/main.go b/cmd/feed2mdir/main.go
index 2699d6cc6e01aa6c2cfbac407eef2e0f39450e6a..815b7d17c8041dad262d0bbc9d250c0ef2e3bab7 100644 (file)
--- a/cmd/feed2mdir/main.go
+++ b/cmd/feed2mdir/main.go
@@ -34,7 +34,7 @@ import (
 )
 
 func main() {
 )
 
 func main() {

-       maxEntries := flag.Uint("max-entries", 100, "Max entries to process")
+       maxEntries := flag.Uint("max-entries", 0, "Max entries to process (0=unlimited)")

        flag.Parse()
        mdir := flag.Arg(0)
        fp := gofeed.NewParser()
        flag.Parse()
        mdir := flag.Arg(0)
        fp := gofeed.NewParser()


@@ -60,7 +60,7 @@ func main() {
        now := time.Now()
        latest := &time.Time{}
        for n, item := range feed.Items {
        now := time.Now()
        latest := &time.Time{}
        for n, item := range feed.Items {

-               if n == int(*maxEntries) {
+               if *maxEntries > 0 && n == int(*maxEntries) {

                        break
                }
                when = nil
                        break
                }
                when = nil

diff --git a/cmd/parse.sh b/cmd/parse.sh
index 9ba9d7c0c3ddb90a7c768191800a7768333d8043..3af3258892c8bb09d2b430aa727b3d66c5a29c3a 100755 (executable)
--- a/cmd/parse.sh
+++ b/cmd/parse.sh
@@ -5,6 +5,7 @@ cd "$1"
 [ -s parse.hash ] && hash_our=`cat parse.hash` || :
 [ -s download.hash ] && hash_their=`cat download.hash` || :
 [ "$hash_our" != "$hash_their" ] || exit 0
 [ -s parse.hash ] && hash_our=`cat parse.hash` || :
 [ -s download.hash ] && hash_their=`cat download.hash` || :
 [ "$hash_our" != "$hash_their" ] || exit 0

-zstd -d < feed.zst | $cmds/feed2mdir/feed2mdir . > title.tmp
+[ -s max ] && max=`cat max` || max=${FEEDER_MAX_ITEMS:-100}
+zstd -d < feed.zst | $cmds/feed2mdir/feed2mdir -max-entries $max . > title.tmp

 mv title.tmp title
 echo $hash_their > parse.hash
 mv title.tmp title
 echo $hash_their > parse.hash

diff --git a/doc/storage.texi b/doc/storage.texi
index e01525438b6e756c4d49a08202ffd05954f2aa28..162c23dd52a2716a1436f2897bba055c16cedcfd 100644 (file)
--- a/doc/storage.texi
+++ b/doc/storage.texi
@@ -11,6 +11,10 @@ contains:
 File with the URL of the feed. This is the only file you have to
 manually deal with.
 
 File with the URL of the feed. This is the only file you have to
 manually deal with.
 

+@item max
+If may contain maximal number of messages per current feed to keep and
+process.
+

 @item etag, hdr, out
 Those files are used by @command{curl} to keep the content, its proper
 @code{mtime} (for @code{If-Modified-Since} header generation),
 @item etag, hdr, out
 Those files are used by @command{curl} to keep the content, its proper
 @code{mtime} (for @code{If-Modified-Since} header generation),

diff --git a/doc/usage.texi b/doc/usage.texi
index 5fa9f800a4352be11ced6ed591e378df4a8e0984..b59afeafc14f0945cc7d2b7d767c1c62c638543f 100644 (file)
--- a/doc/usage.texi
+++ b/doc/usage.texi
@@ -157,16 +157,20 @@ message flags display and adding name of the feed in parenthesis.
 
 @item Cleanup excess number of messages
 
 
 @item Cleanup excess number of messages
 

+By default (@env{$FEEDER_MAX_ITEMS}) only 100 entries are processed.
+Parser only appends them, but does not remove obsolete ones.
+

 @example
 $ ./feeds-clear.zsh
 @end example
 
 @example
 $ ./feeds-clear.zsh
 @end example
 

-That will remove all messages in all feeds @file{cur/} directory that is
-not first hundred of ones, ordered by @code{mtime}. Pay attention that
-@file{new/} directory is not touched, so you won't loose completely new
-and unread messages when you are on vacation and left @command{cron}-ed
-workers. @command{cmd/feed2mdir/feed2mdir} command by default has
-@option{-max-entries 100} option set.
+will clear everything exceeding the quantity limit. You can set that
+limit on per-feed basis. For example @code{echo 50 > feed/FEED/max}.
+0 means no limit and keep all the messages.
+
+Pay attention that @file{new/} directory is not touched, so you won't
+loose completely new and unread messages when you are on vacation and
+left @command{cron}-ed workers.

 
 @item If you want to clean download state
 
 
 @item If you want to clean download state
 

diff --git a/feeds-clear.zsh b/feeds-clear.zsh
index 2e5002f4a7e1915895f19d339ba809b17a6a14d1..c1a55aad69a220aae9ad8c8af7c7db218ee4a132 100755 (executable)
--- a/feeds-clear.zsh
+++ b/feeds-clear.zsh
@@ -1,8 +1,8 @@
 #!/usr/bin/env zsh
 set -e
 setopt EXTENDED_GLOB
 #!/usr/bin/env zsh
 set -e
 setopt EXTENDED_GLOB

-for f (feeds/**/cur) {
-    pushd $f
-    rm -fv *(Nom[101,-1]) || :
-    popd
+for f (feeds/*) {
+    [[ -s $f/max ]] && max=`cat $f/max` || max=${FEEDER_MAX_ITEMS:-100}
+    (( max++ ))
+    [[ $max -eq 1 ]] || rm -fv $f/cur/*(Nom[$max,-1])

 }
 }