]> Sergey Matveev's repositories - feeder.git/blob - cmd/feed2mdir/main.go
8ee07d0cab392dbf1fc7727b810735075a4e1ebf
[feeder.git] / cmd / feed2mdir / main.go
1 /*
2 go.stargrave.org/feeder  -- newsfeeds aggregator
3 Copyright (C) 2022 Sergey Matveev <stargrave@stargrave.org>
4
5 This program is free software: you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation, version 3 of the License.
8
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12 GNU General Public License for more details.
13
14 You should have received a copy of the GNU General Public License
15 along with this program.  If not, see <http://www.gnu.org/licenses/>.
16 */
17
18 package main
19
20 import (
21         "crypto/sha512"
22         "encoding/base64"
23         "encoding/hex"
24         "flag"
25         "fmt"
26         "log"
27         "mime"
28         "os"
29         "path"
30         "strings"
31         "time"
32
33         "github.com/mmcdole/gofeed"
34 )
35
36 func main() {
37         maxEntries := flag.Uint("max-entries", 100, "Max entries to process")
38         flag.Parse()
39         mdir := flag.Arg(0)
40         fp := gofeed.NewParser()
41         feed, err := fp.Parse(os.Stdin)
42         if err != nil {
43                 log.Fatalln(err)
44         }
45
46         guids := make(map[string]struct{}, len(feed.Items))
47         useGUID := true
48         for _, item := range feed.Items {
49                 if _, exists := guids[item.GUID]; exists {
50                         useGUID = false
51                         break
52                 } else {
53                         guids[item.GUID] = struct{}{}
54                 }
55         }
56
57         h := sha512.New()
58         news := 0
59         var when *time.Time
60         now := time.Now()
61         for n, item := range feed.Items {
62                 if n == int(*maxEntries) {
63                         break
64                 }
65                 when = nil
66                 if item.PublishedParsed != nil {
67                         when = item.PublishedParsed
68                 } else if item.UpdatedParsed != nil {
69                         when = item.UpdatedParsed
70                 } else {
71                         when = &now
72                 }
73                 var what string
74                 if len(item.Content) == 0 {
75                         what = item.Description
76                 } else {
77                         what = item.Content
78                 }
79                 what = strings.TrimPrefix(what, "<![CDATA[")
80                 what = strings.TrimSuffix(what, "]]>")
81                 h.Reset()
82                 if useGUID {
83                         h.Write([]byte(item.GUID))
84                 } else {
85                         h.Write([]byte(item.Title))
86                         h.Write([]byte{0})
87                         h.Write([]byte(what))
88                 }
89                 fn := hex.EncodeToString(h.Sum(nil)[:sha512.Size/2])
90                 exists := false
91                 for _, d := range []string{"cur", "new"} {
92                         entries, err := os.ReadDir(path.Join(mdir, d))
93                         if err != nil {
94                                 log.Fatalln(err)
95                         }
96                         for _, entry := range entries {
97                                 if strings.HasPrefix(entry.Name(), fn) {
98                                         exists = true
99                                         break
100                                 }
101                         }
102                 }
103                 if exists {
104                         continue
105                 }
106                 fn = path.Join(mdir, "new", fn)
107                 fd, err := os.OpenFile(fn, os.O_WRONLY|os.O_CREATE|os.O_EXCL, os.FileMode(0666))
108                 if err != nil {
109                         log.Fatalln(err)
110                 }
111                 fd.WriteString("From: \"" + feed.Title + "\" <feeder@go.stargrave.org>\n")
112                 fd.WriteString("Date: " + when.UTC().Format(time.RFC1123Z) + "\n")
113                 fd.WriteString("Subject: " + mime.BEncoding.Encode("UTF-8", item.Title) + "\n")
114                 fd.WriteString("MIME-Version: 1.0\n")
115                 fd.WriteString("Content-Type: text/html; charset=utf-8\n")
116                 fd.WriteString("Content-Transfer-Encoding: base64\n")
117                 for _, link := range item.Links {
118                         fd.WriteString("X-URL: " + link + "\n")
119                 }
120                 for _, author := range item.Authors {
121                         fd.WriteString("X-Author: " + author.Name + "\n")
122                 }
123                 for _, cat := range item.Categories {
124                         fd.WriteString("X-Category: " + cat + "\n")
125                 }
126                 fd.WriteString("\n")
127                 what = base64.StdEncoding.EncodeToString([]byte(what))
128                 for i := 0; i < len(what); i += 72 {
129                         b := i + 72
130                         if b > len(what) {
131                                 b = len(what)
132                         }
133                         fd.WriteString(what[i:b] + "\n")
134                 }
135                 fd.Close()
136                 if err = os.Chtimes(fn, *when, *when); err != nil {
137                         log.Fatalln(err)
138                 }
139                 news++
140         }
141         when = nil
142         if feed.PublishedParsed != nil {
143                 when = feed.PublishedParsed
144         } else if feed.UpdatedParsed != nil {
145                 when = feed.UpdatedParsed
146         }
147         if when != nil {
148                 for _, d := range []string{"cur", "new"} {
149                         if err = os.Chtimes(path.Join(mdir, d), *when, *when); err != nil {
150                                 log.Fatalln(err)
151                         }
152                 }
153         }
154         fmt.Println(feed.Title)
155 }