cmd/feed2mdir/main.go

   1 /*
   2 feeder  -- newsfeeds aggregator
   3 Copyright (C) 2022-2023 Sergey Matveev <stargrave@stargrave.org>
   4
   5 This program is free software: you can redistribute it and/or modify
   6 it under the terms of the GNU General Public License as published by
   7 the Free Software Foundation, version 3 of the License.
   8
   9 This program is distributed in the hope that it will be useful,
  10 but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 GNU General Public License for more details.
  13
  14 You should have received a copy of the GNU General Public License
  15 along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16 */
  17
  18 package main
  19
  20 import (
  21         "crypto/sha512"
  22         "encoding/base64"
  23         "encoding/hex"
  24         "flag"
  25         "fmt"
  26         "log"
  27         "mime"
  28         "os"
  29         "path"
  30         "path/filepath"
  31         "strings"
  32         "time"
  33
  34         "github.com/mmcdole/gofeed"
  35 )
  36
  37 func main() {
  38         maxEntries := flag.Uint("max-entries", 0, "Max entries to process (0=unlimited)")
  39         flag.Parse()
  40         mdir := flag.Arg(0)
  41         fp := gofeed.NewParser()
  42         feed, err := fp.Parse(os.Stdin)
  43         if err != nil {
  44                 log.Fatalln(err)
  45         }
  46
  47         guids := make(map[string]struct{}, len(feed.Items))
  48         useGUID := true
  49         for _, item := range feed.Items {
  50                 if _, exists := guids[item.GUID]; exists {
  51                         useGUID = false
  52                         break
  53                 } else {
  54                         guids[item.GUID] = struct{}{}
  55                 }
  56         }
  57
  58         feedTitle := feed.Title
  59         if len(feedTitle) == 0 {
  60                 feedTitle, err = filepath.Abs(mdir)
  61                 if err != nil {
  62                         log.Fatalln(err)
  63                 }
  64                 feedTitle = path.Base(feedTitle)
  65         }
  66
  67         h := sha512.New()
  68         news := 0
  69         var when *time.Time
  70         now := time.Now()
  71         latest := &time.Time{}
  72         for n, item := range feed.Items {
  73                 if *maxEntries > 0 && n == int(*maxEntries) {
  74                         break
  75                 }
  76                 when = nil
  77                 if item.PublishedParsed != nil {
  78                         when = item.PublishedParsed
  79                 } else if item.UpdatedParsed != nil {
  80                         when = item.UpdatedParsed
  81                 } else {
  82                         when = &now
  83                 }
  84                 if latest.Before(*when) {
  85                         latest = when
  86                 }
  87                 var what string
  88                 if len(item.Content) == 0 {
  89                         what = item.Description
  90                 } else {
  91                         what = item.Content
  92                 }
  93                 what = strings.TrimPrefix(what, "<![CDATA[")
  94                 what = strings.TrimSuffix(what, "]]>")
  95                 h.Reset()
  96                 if useGUID {
  97                         h.Write([]byte(item.GUID))
  98                 } else {
  99                         h.Write([]byte(item.Title))
 100                         h.Write([]byte{0})
 101                         h.Write([]byte(what))
 102                 }
 103                 fn := hex.EncodeToString(h.Sum(nil)[:sha512.Size/2])
 104                 exists := false
 105                 for _, d := range []string{"cur", "new"} {
 106                         entries, err := os.ReadDir(path.Join(mdir, d))
 107                         if err != nil {
 108                                 log.Fatalln(err)
 109                         }
 110                         for _, entry := range entries {
 111                                 if strings.HasPrefix(entry.Name(), fn) {
 112                                         exists = true
 113                                         break
 114                                 }
 115                         }
 116                 }
 117                 if exists {
 118                         continue
 119                 }
 120                 fn = path.Join(mdir, "new", fn)
 121                 fd, err := os.OpenFile(fn, os.O_WRONLY|os.O_CREATE|os.O_EXCL, os.FileMode(0666))
 122                 if err != nil {
 123                         log.Fatalln(err)
 124                 }
 125                 fd.WriteString("From: \"" + feedTitle + "\" <feeder@localhost>\n")
 126                 fd.WriteString("Date: " + when.UTC().Format(time.RFC1123Z) + "\n")
 127                 fd.WriteString("Subject: " + mime.BEncoding.Encode("UTF-8", item.Title) + "\n")
 128                 fd.WriteString("MIME-Version: 1.0\n")
 129                 fd.WriteString("Content-Type: text/html; charset=utf-8\n")
 130                 fd.WriteString("Content-Transfer-Encoding: base64\n")
 131                 for _, author := range item.Authors {
 132                         if len(author.Name) > 0 {
 133                                 fd.WriteString("X-Author: " + author.Name + "\n")
 134                         }
 135                 }
 136                 for _, link := range item.Links {
 137                         fd.WriteString("X-URL: " + link + "\n")
 138                 }
 139                 for _, enc := range item.Enclosures {
 140                         fd.WriteString("X-Enclosure: " + enc.URL + "\n")
 141                 }
 142                 if len(item.Categories) > 0 {
 143                         fd.WriteString("X-Categories: " + strings.Join(item.Categories, ", ") + "\n")
 144                 }
 145                 fd.WriteString("\n")
 146                 what = base64.StdEncoding.EncodeToString([]byte(what))
 147                 for i := 0; i < len(what); i += 72 {
 148                         b := i + 72
 149                         if b > len(what) {
 150                                 b = len(what)
 151                         }
 152                         fd.WriteString(what[i:b] + "\n")
 153                 }
 154                 fd.Close()
 155                 if err = os.Chtimes(fn, *when, *when); err != nil {
 156                         log.Fatalln(err)
 157                 }
 158                 news++
 159         }
 160         for _, d := range []string{"cur", "new"} {
 161                 if err = os.Chtimes(path.Join(mdir, d), *latest, *latest); err != nil {
 162                         log.Fatalln(err)
 163                 }
 164         }
 165         fmt.Println(feedTitle)
 166 }