cmd/feed2mdir/main.go

   1 // feeder -- newsfeeds aggregator
   2 // Copyright (C) 2022-2024 Sergey Matveev <stargrave@stargrave.org>
   3 //
   4 // This program is free software: you can redistribute it and/or modify
   5 // it under the terms of the GNU General Public License as published by
   6 // the Free Software Foundation, version 3 of the License.
   7 //
   8 // This program is distributed in the hope that it will be useful,
   9 // but WITHOUT ANY WARRANTY; without even the implied warranty of
  10 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11 // GNU General Public License for more details.
  12 //
  13 // You should have received a copy of the GNU General Public License
  14 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
  15
  16 package main
  17
  18 import (
  19         "crypto/sha512"
  20         "encoding/base64"
  21         "encoding/hex"
  22         "flag"
  23         "fmt"
  24         "log"
  25         "mime"
  26         "os"
  27         "path"
  28         "path/filepath"
  29         "strings"
  30         "time"
  31
  32         "github.com/mmcdole/gofeed"
  33 )
  34
  35 func main() {
  36         maxEntries := flag.Uint("max-entries", 0, "Max entries to process (0=unlimited)")
  37         flag.Parse()
  38         mdir := flag.Arg(0)
  39         fp := gofeed.NewParser()
  40         feed, err := fp.Parse(os.Stdin)
  41         if err != nil {
  42                 log.Fatalln(err)
  43         }
  44
  45         guids := make(map[string]struct{}, len(feed.Items))
  46         useGUID := true
  47         for _, item := range feed.Items {
  48                 if _, exists := guids[item.GUID]; exists {
  49                         useGUID = false
  50                         break
  51                 } else {
  52                         guids[item.GUID] = struct{}{}
  53                 }
  54         }
  55
  56         feedTitle := feed.Title
  57         if len(feedTitle) == 0 {
  58                 feedTitle, err = filepath.Abs(mdir)
  59                 if err != nil {
  60                         log.Fatalln(err)
  61                 }
  62                 feedTitle = path.Base(feedTitle)
  63         }
  64
  65         h := sha512.New()
  66         news := 0
  67         var when *time.Time
  68         now := time.Now()
  69         latest := &time.Time{}
  70         for n, item := range feed.Items {
  71                 if *maxEntries > 0 && n == int(*maxEntries) {
  72                         break
  73                 }
  74                 when = nil
  75                 if item.PublishedParsed != nil {
  76                         when = item.PublishedParsed
  77                 } else if item.UpdatedParsed != nil {
  78                         when = item.UpdatedParsed
  79                 } else {
  80                         when = &now
  81                 }
  82                 if latest.Before(*when) {
  83                         latest = when
  84                 }
  85                 var what string
  86                 if len(item.Content) > len(item.Description) {
  87                         what = item.Content
  88                 } else {
  89                         what = item.Description
  90                 }
  91                 if media, ok := item.Extensions["media"]; ok {
  92                         if mediagroups, ok := media["group"]; ok {
  93                                 if len(mediagroups) == 1 {
  94                                         if mediadescription, ok := mediagroups[0].Children["description"]; ok {
  95                                                 if len(mediadescription[0].Value) > len(what) {
  96                                                         what = mediadescription[0].Value
  97                                                 }
  98                                         }
  99                                 }
 100                         }
 101                 }
 102                 what = strings.TrimPrefix(what, "<![CDATA[")
 103                 what = strings.TrimSuffix(what, "]]>")
 104                 h.Reset()
 105                 if useGUID {
 106                         h.Write([]byte(item.GUID))
 107                 } else {
 108                         h.Write([]byte(item.Title))
 109                         h.Write([]byte{0})
 110                         h.Write([]byte(what))
 111                 }
 112                 fn := hex.EncodeToString(h.Sum(nil)[:sha512.Size/2])
 113                 exists := false
 114                 for _, d := range []string{"cur", "new"} {
 115                         entries, err := os.ReadDir(path.Join(mdir, d))
 116                         if err != nil {
 117                                 log.Fatalln(err)
 118                         }
 119                         for _, entry := range entries {
 120                                 if strings.HasPrefix(entry.Name(), fn) {
 121                                         exists = true
 122                                         break
 123                                 }
 124                         }
 125                 }
 126                 if exists {
 127                         continue
 128                 }
 129                 fn = path.Join(mdir, "new", fn)
 130                 fd, err := os.OpenFile(fn, os.O_WRONLY|os.O_CREATE|os.O_EXCL, os.FileMode(0666))
 131                 if err != nil {
 132                         log.Fatalln(err)
 133                 }
 134                 fd.WriteString("From: \"" + feedTitle + "\" <feeder@localhost>\n")
 135                 fd.WriteString("Date: " + when.UTC().Format(time.RFC1123Z) + "\n")
 136                 fd.WriteString("Subject: " + mime.BEncoding.Encode("UTF-8", item.Title) + "\n")
 137                 fd.WriteString("MIME-Version: 1.0\n")
 138                 fd.WriteString("Content-Type: text/html; charset=utf-8\n")
 139                 fd.WriteString("Content-Transfer-Encoding: base64\n")
 140                 for _, author := range item.Authors {
 141                         if len(author.Name) > 0 {
 142                                 fd.WriteString("X-Author: " + author.Name + "\n")
 143                         }
 144                 }
 145                 for _, link := range item.Links {
 146                         fd.WriteString("X-URL: " + link + "\n")
 147                 }
 148                 for _, enc := range item.Enclosures {
 149                         fd.WriteString("X-Enclosure: " + enc.URL + "\n")
 150                 }
 151                 if len(item.Categories) > 0 {
 152                         fd.WriteString("X-Categories: " + strings.Join(item.Categories, ", ") + "\n")
 153                 }
 154                 fd.WriteString("\n")
 155                 what = base64.StdEncoding.EncodeToString([]byte(what))
 156                 for i := 0; i < len(what); i += 72 {
 157                         b := i + 72
 158                         if b > len(what) {
 159                                 b = len(what)
 160                         }
 161                         fd.WriteString(what[i:b] + "\n")
 162                 }
 163                 fd.Close()
 164                 if err = os.Chtimes(fn, *when, *when); err != nil {
 165                         log.Fatalln(err)
 166                 }
 167                 news++
 168         }
 169         for _, d := range []string{"cur", "new"} {
 170                 if err = os.Chtimes(path.Join(mdir, d), *latest, *latest); err != nil {
 171                         log.Fatalln(err)
 172                 }
 173         }
 174         fmt.Println(feedTitle)
 175 }