/*
-go.stargrave.org/feeder -- newsfeeds aggregator
+feeder -- newsfeeds aggregator
Copyright (C) 2022 Sergey Matveev <stargrave@stargrave.org>
This program is free software: you can redistribute it and/or modify
"mime"
"os"
"path"
+ "path/filepath"
"strings"
"time"
)
func main() {
- maxEntries := flag.Uint("max-entries", 100, "Max entries to process")
+ maxEntries := flag.Uint("max-entries", 0, "Max entries to process (0=unlimited)")
flag.Parse()
mdir := flag.Arg(0)
fp := gofeed.NewParser()
if err != nil {
log.Fatalln(err)
}
+
+ guids := make(map[string]struct{}, len(feed.Items))
+ useGUID := true
+ for _, item := range feed.Items {
+ if _, exists := guids[item.GUID]; exists {
+ useGUID = false
+ break
+ } else {
+ guids[item.GUID] = struct{}{}
+ }
+ }
+
+ feedTitle := feed.Title
+ if len(feedTitle) == 0 {
+ feedTitle, err = filepath.Abs(mdir)
+ if err != nil {
+ log.Fatalln(err)
+ }
+ feedTitle = path.Base(feedTitle)
+ }
+
+ h := sha512.New()
news := 0
var when *time.Time
now := time.Now()
- max := int(*maxEntries) - 1
+ latest := &time.Time{}
for n, item := range feed.Items {
- if n == max {
+ if *maxEntries > 0 && n == int(*maxEntries) {
break
}
when = nil
- h := sha512.New()
- h.Write([]byte(item.Title))
- h.Write([]byte{0})
- if item.UpdatedParsed != nil {
- when = item.UpdatedParsed
- h.Write([]byte(when.String()))
- } else if item.PublishedParsed != nil {
+ if item.PublishedParsed != nil {
when = item.PublishedParsed
- h.Write([]byte(when.String()))
+ } else if item.UpdatedParsed != nil {
+ when = item.UpdatedParsed
} else {
when = &now
}
- fn := hex.EncodeToString(h.Sum(nil)[:sha512.Size/2])
+ if latest.Before(*when) {
+ latest = when
+ }
var what string
if len(item.Content) == 0 {
what = item.Description
} else {
what = item.Content
}
+ what = strings.TrimPrefix(what, "<![CDATA[")
+ what = strings.TrimSuffix(what, "]]>")
+ h.Reset()
+ if useGUID {
+ h.Write([]byte(item.GUID))
+ } else {
+ h.Write([]byte(item.Title))
+ h.Write([]byte{0})
+ h.Write([]byte(what))
+ }
+ fn := hex.EncodeToString(h.Sum(nil)[:sha512.Size/2])
exists := false
for _, d := range []string{"cur", "new"} {
entries, err := os.ReadDir(path.Join(mdir, d))
if err != nil {
log.Fatalln(err)
}
- fd.WriteString("From: \"" + feed.Title + "\" <feeder@go.stargrave.org>\n")
- fd.WriteString("Date: " + when.Format(time.RFC1123Z) + "\n")
+ fd.WriteString("From: \"" + feedTitle + "\" <feeder@localhost>\n")
+ fd.WriteString("Date: " + when.UTC().Format(time.RFC1123Z) + "\n")
fd.WriteString("Subject: " + mime.BEncoding.Encode("UTF-8", item.Title) + "\n")
fd.WriteString("MIME-Version: 1.0\n")
fd.WriteString("Content-Type: text/html; charset=utf-8\n")
fd.WriteString("Content-Transfer-Encoding: base64\n")
+ for _, author := range item.Authors {
+ if len(author.Name) > 0 {
+ fd.WriteString("X-Author: " + author.Name + "\n")
+ }
+ }
for _, link := range item.Links {
fd.WriteString("X-URL: " + link + "\n")
}
- for _, author := range item.Authors {
- fd.WriteString("X-Author: " + author.Name + "\n")
+ for _, enc := range item.Enclosures {
+ fd.WriteString("X-Enclosure: " + enc.URL + "\n")
}
- for _, cat := range item.Categories {
- fd.WriteString("X-Category: " + cat + "\n")
+ if len(item.Categories) > 0 {
+ fd.WriteString("X-Categories: " + strings.Join(item.Categories, ", ") + "\n")
}
fd.WriteString("\n")
what = base64.StdEncoding.EncodeToString([]byte(what))
}
news++
}
- when = nil
- if feed.UpdatedParsed != nil {
- when = feed.UpdatedParsed
- } else if feed.PublishedParsed != nil {
- when = feed.PublishedParsed
- }
- if when != nil {
- for _, d := range []string{"cur", "new"} {
- if err = os.Chtimes(path.Join(mdir, d), *when, *when); err != nil {
- log.Fatalln(err)
- }
+ for _, d := range []string{"cur", "new"} {
+ if err = os.Chtimes(path.Join(mdir, d), *latest, *latest); err != nil {
+ log.Fatalln(err)
}
}
- fmt.Println(feed.Title)
+ fmt.Println(feedTitle)
}