Per-feed max number of entries

[feeder.git] / cmd / feed2mdir / main.go

diff --git a/cmd/feed2mdir/main.go b/cmd/feed2mdir/main.go
index cac223700db4a7dae8ec28300db276e7f7460945..815b7d17c8041dad262d0bbc9d250c0ef2e3bab7 100644 (file)
--- a/cmd/feed2mdir/main.go
+++ b/cmd/feed2mdir/main.go
@@ -34,7 +34,7 @@ import (
 )
 
 func main() {
-       maxEntries := flag.Uint("max-entries", 100, "Max entries to process")
+       maxEntries := flag.Uint("max-entries", 0, "Max entries to process (0=unlimited)")
        flag.Parse()
        mdir := flag.Arg(0)
        fp := gofeed.NewParser()
@@ -42,12 +42,25 @@ func main() {
        if err != nil {
                log.Fatalln(err)
        }
+
+       guids := make(map[string]struct{}, len(feed.Items))
+       useGUID := true
+       for _, item := range feed.Items {
+               if _, exists := guids[item.GUID]; exists {
+                       useGUID = false
+                       break
+               } else {
+                       guids[item.GUID] = struct{}{}
+               }
+       }
+
+       h := sha512.New()
        news := 0
        var when *time.Time
        now := time.Now()
-       max := int(*maxEntries) - 1
+       latest := &time.Time{}
        for n, item := range feed.Items {
-               if n == max {
+               if *maxEntries > 0 && n == int(*maxEntries) {
                        break
                }
                when = nil
@@ -58,13 +71,26 @@ func main() {
                } else {
                        when = &now
                }
-               fn := hex.EncodeToString(h.Sum(nil)[:sha512.Size/2])
+               if latest.Before(*when) {
+                       latest = when
+               }
                var what string
                if len(item.Content) == 0 {
                        what = item.Description
                } else {
                        what = item.Content
                }
+               what = strings.TrimPrefix(what, "<![CDATA[")
+               what = strings.TrimSuffix(what, "]]>")
+               h.Reset()
+               if useGUID {
+                       h.Write([]byte(item.GUID))
+               } else {
+                       h.Write([]byte(item.Title))
+                       h.Write([]byte{0})
+                       h.Write([]byte(what))
+               }
+               fn := hex.EncodeToString(h.Sum(nil)[:sha512.Size/2])
                exists := false
                for _, d := range []string{"cur", "new"} {
                        entries, err := os.ReadDir(path.Join(mdir, d))
@@ -92,14 +118,19 @@ func main() {
                fd.WriteString("MIME-Version: 1.0\n")
                fd.WriteString("Content-Type: text/html; charset=utf-8\n")
                fd.WriteString("Content-Transfer-Encoding: base64\n")
+               for _, author := range item.Authors {
+                       if len(author.Name) > 0 {
+                               fd.WriteString("X-Author: " + author.Name + "\n")
+                       }
+               }
                for _, link := range item.Links {
                        fd.WriteString("X-URL: " + link + "\n")
                }
-               for _, author := range item.Authors {
-                       fd.WriteString("X-Author: " + author.Name + "\n")
+               for _, enc := range item.Enclosures {
+                       fd.WriteString("X-Enclosure: " + enc.URL + "\n")
                }
-               for _, cat := range item.Categories {
-                       fd.WriteString("X-Category: " + cat + "\n")
+               if len(item.Categories) > 0 {
+                       fd.WriteString("X-Categories: " + strings.Join(item.Categories, ", ") + "\n")
                }
                fd.WriteString("\n")
                what = base64.StdEncoding.EncodeToString([]byte(what))
@@ -116,17 +147,9 @@ func main() {
                }
                news++
        }
-       when = nil
-       if feed.PublishedParsed != nil {
-               when = feed.PublishedParsed
-       } else if feed.UpdatedParsed != nil {
-               when = feed.UpdatedParsed
-       }
-       if when != nil {
-               for _, d := range []string{"cur", "new"} {
-                       if err = os.Chtimes(path.Join(mdir, d), *when, *when); err != nil {
-                               log.Fatalln(err)
-                       }
+       for _, d := range []string{"cur", "new"} {
+               if err = os.Chtimes(path.Join(mdir, d), *latest, *latest); err != nil {
+                       log.Fatalln(err)
                }
        }
        fmt.Println(feed.Title)