From d1d5065147a0214fe737a9c493da810041c32c71 Mon Sep 17 00:00:00 2001 From: Sergey Matveev Date: Sat, 26 Sep 2020 12:14:56 +0300 Subject: [PATCH] Topics support --- README.texi | 31 +++++++-- cmd/sgblog/atom/atom.go | 66 +++++++++++++++++++ cmd/sgblog/gopher.go | 138 ++++++++++++++++++++++++++-------------- cmd/sgblog/http.go | 138 +++++++++++++++++++++++++++++++++++----- cmd/sgblog/main.go | 25 ++++++++ cmd/sgblog/topics.go | 131 ++++++++++++++++++++++++++++++++++++++ common.go | 2 +- 7 files changed, 462 insertions(+), 69 deletions(-) create mode 100644 cmd/sgblog/atom/atom.go create mode 100644 cmd/sgblog/topics.go diff --git a/README.texi b/README.texi index c2b5887..a0309d8 100644 --- a/README.texi +++ b/README.texi @@ -25,6 +25,7 @@ Its main competitive features: @item Uses @url{https://en.wikipedia.org/wiki/Inetd, inetd} interface for working as @url{https://en.wikipedia.org/wiki/Gopher_(protocol), Gopher} server +@item Topics (tags/categories) support @item Supports on the fly generation of @url{https://en.wikipedia.org/wiki/Atom_(feed), Atom} feeds for posts, comments and per-post comments @@ -33,9 +34,9 @@ Its main competitive features: copy of your blog/phlog! @end itemize -All of that, except for comments and phlog, could be achieved with some -Git viewer like @url{https://git.zx2c4.com/cgit/about/, cgit}. But -SGBlog also is able to: +All of that, except for comments, topics and phlog, could be achieved +with some Git viewer like @url{https://git.zx2c4.com/cgit/about/, cgit}. +But SGBlog also is able to: @itemize @item Convert URLs to clickable links @@ -56,6 +57,7 @@ see the file COPYING for copying conditions. @menu * Comments:: +* Topics:: * Installation:: * Configuration:: @end menu @@ -67,7 +69,7 @@ Comments are posted through the email interface, just by sending the message to special address. For example: @example -mutt "mailto:comment@@blog.example.com?subject=576540a5b98517b46d0efc791bb90b9121bf147e" < + + + Redirect to non-gopher URL + + +Redirecting to %s... + + +`, selector, selector, selector) + } else if sha1DigestRe.MatchString(selector) { + commit, err := repo.CommitObject(plumbing.NewHash(selector[1:])) + if err != nil { + log.Fatalln(err) + } + tmpl := template.Must(template.New("entry").Parse(TmplGopherEntry)) + err = tmpl.Execute(os.Stdout, struct { + Commit *object.Commit + When string + Cfg *Cfg + Note string + Comments []string + Topics []string + Version string + }{ + Commit: commit, + When: commit.Author.When.Format(sgblog.WhenFmt), + Cfg: cfg, + Note: string(getNote(notesTree, commit.Hash)), + Comments: parseComments(getNote(commentsTree, commit.Hash)), + Topics: parseTopics(getNote(topicsTree, commit.Hash)), + Version: sgblog.Version, + }) + if err != nil { + log.Fatalln(err) + } + } else if selectorParts[len(selectorParts)-2] == "offset" { + offset, err := strconv.Atoi(selectorParts[len(selectorParts)-1]) if err != nil { log.Fatalln(err) } @@ -124,15 +168,42 @@ func serveGopher() { if err != nil { log.Fatalln(err) } - for i := 0; i < offset; i++ { - if _, err = repoLog.Next(); err != nil { - break + topicsCache, err := getTopicsCache(cfg, repoLog) + if err != nil { + log.Fatalln(err) + } + repoLog, err = repo.Log(&git.LogOptions{From: *headHash}) + if err != nil { + log.Fatalln(err) + } + + var topic string + if len(selectorParts) == 3 { + topic = selectorParts[0] + } + var commits CommitIterNext + if topic == "" { + for i := 0; i < offset; i++ { + if _, err = repoLog.Next(); err != nil { + break + } } + commits = repoLog + } else { + hashes := topicsCache[topic] + if hashes == nil { + log.Fatalln(errors.New("no posts with that topic")) + } + if len(hashes) > offset { + hashes = hashes[offset:] + } + commits = &HashesIter{hashes} } + logEnded := false entries := make([]TableMenuEntry, 0, PageEntries) for i := 0; i < PageEntries; i++ { - commit, err := repoLog.Next() + commit, err := commits.Next() if err != nil { logEnded = true break @@ -143,65 +214,38 @@ func serveGopher() { Title: lines[0], LinesNum: len(lines) - 2, CommentsNum: len(parseComments(getNote(commentsTree, commit.Hash))), + Topics: parseTopics(getNote(topicsTree, commit.Hash)), }) } tmpl := template.Must(template.New("menu").Parse(TmplGopherMenu)) + offsetPrev := offset - PageEntries + if offsetPrev < 0 { + offsetPrev = 0 + } err = tmpl.Execute(os.Stdout, struct { Cfg *Cfg + Topic string Offset int OffsetPrev int OffsetNext int LogEnded bool Entries []TableMenuEntry + Topics []string Version string }{ Cfg: cfg, + Topic: topic, Offset: offset, - OffsetPrev: offset - PageEntries, + OffsetPrev: offsetPrev, OffsetNext: offset + PageEntries, LogEnded: logEnded, Entries: entries, + Topics: topicsCache.Topics(), Version: sgblog.Version, }) if err != nil { log.Fatalln(err) } - } else if strings.HasPrefix(selector, "URL:") { - selector = selector[len("URL:"):] - fmt.Printf(` - - - Redirect to non-gopher URL - - -Redirecting to %s... - - -`, selector, selector, selector) - } else if sha1DigestRe.MatchString(selector) { - commit, err := repo.CommitObject(plumbing.NewHash(selector[1:])) - if err != nil { - log.Fatalln(err) - } - tmpl := template.Must(template.New("entry").Parse(TmplGopherEntry)) - err = tmpl.Execute(os.Stdout, struct { - Commit *object.Commit - When string - Cfg *Cfg - Note string - Comments []string - Version string - }{ - Commit: commit, - When: commit.Author.When.Format(sgblog.WhenFmt), - Cfg: cfg, - Note: string(getNote(notesTree, commit.Hash)), - Comments: parseComments(getNote(commentsTree, commit.Hash)), - Version: sgblog.Version, - }) - if err != nil { - log.Fatalln(err) - } } else { log.Fatalln(errors.New("unknown selector")) } diff --git a/cmd/sgblog/http.go b/cmd/sgblog/http.go index 6d09420..854ebe6 100644 --- a/cmd/sgblog/http.go +++ b/cmd/sgblog/http.go @@ -42,8 +42,8 @@ import ( "github.com/go-git/go-git/v5/plumbing/object" "github.com/hjson/hjson-go" "go.stargrave.org/sgblog" + "go.stargrave.org/sgblog/cmd/sgblog/atom" "golang.org/x/crypto/blake2b" - "golang.org/x/tools/blog/atom" ) const ( @@ -53,23 +53,29 @@ const ( - {{.Cfg.Title}} ({{.Offset}}-{{.OffsetNext}}) + {{.Cfg.Title}} {{if .Topic}}(topic: {{.Topic}}) {{end}}({{.Offset}}-{{.OffsetNext}}) {{with .Cfg.CSS}}{{end}} {{with .Cfg.Webmaster}}{{end}} {{range .Cfg.GitURLs}}{{end}} - + {{if .CommentsEnabled}}{{end}} - {{if .Offset}}{{end}} - {{if not .LogEnded}}{{end}} + {{if .Offset}}{{end}} + {{if not .LogEnded}}{{end}} {{with .Cfg.AboutURL}}[about]{{end}} {{block "links" .}} -{{if .Offset}}[prev]{{end}} -{{if not .LogEnded}}[next]{{end}} +{{if .Offset}}[prev]{{end}} +{{if not .LogEnded}}[next]{{end}} {{end}} {{- $Cfg := .Cfg -}} +{{if .Topics}}
+Topics: [ALL] +{{range .Topics}}[{{.}}] +{{end}} +{{end}} +{{- $TopicsEnabled := .TopicsEnabled -}} {{- $datePrev := "0001-01-01" -}} @@ -77,11 +83,12 @@ const ( + {{if .TopicsEnabled}}{{end}} {{range .Entries -}} {{- $dateCur := .Commit.Author.When.Format "2006-01-02" -}} {{- if ne $dateCur $datePrev -}} - + {{- $datePrev = $dateCur -}} {{- end -}} @@ -91,6 +98,7 @@ const ( + {{if $TopicsEnabled}}{{end}} {{end}}
L C Linked toTopics
{{$dateCur}}
{{$dateCur}}
{{.LinesNum}} {{if .CommentsNum}}{{.CommentsNum}}{{else}} {{end}} {{if .DomainURLs}}{{range .DomainURLs}} {{.}} {{end}}{{else}} {{end}}{{if .Topics}}{{range .Topics}} {{.}} {{end}}{{else}} {{end}}
{{template "links" .}} @@ -117,6 +125,11 @@ const ( [{{.When}}] [{{.Commit.Hash.String}}] +{{if .Topics}} +
+Topics: {{range .Topics}}[{{.}}]{{end}} +{{end}} +

{{.Title}}

@@ -157,11 +170,13 @@ var (
 type TableEntry struct {
 	Commit      *object.Commit
 	CommentsRaw []byte
+	TopicsRaw   []byte
 	Num         int
 	Title       string
 	LinesNum    int
 	CommentsNum int
 	DomainURLs  []string
+	Topics      []string
 }
 
 type CommentEntry struct {
@@ -236,6 +251,10 @@ func bytes2uuid(b []byte) string {
 	return fmt.Sprintf("%x-%x-%x-%x-%x", raw[0:4], raw[4:6], raw[6:8], raw[8:10], raw[10:])
 }
 
+type CommitIterNext interface {
+	Next() (*object.Commit, error)
+}
+
 func serveHTTP() {
 	cfgPath := os.Getenv("SGBLOG_CFG")
 	if cfgPath == "" {
@@ -349,12 +368,37 @@ func serveHTTP() {
 		if err != nil {
 			makeErr(err)
 		}
+		topicsCache, err := getTopicsCache(cfg, repoLog)
+		if err != nil {
+			makeErr(err)
+		}
+		repoLog, err = repo.Log(&git.LogOptions{From: *headHash})
+		if err != nil {
+			makeErr(err)
+		}
+
 		commitN := 0
-		for i := 0; i < offset; i++ {
-			if _, err = repoLog.Next(); err != nil {
-				break
+		var commits CommitIterNext
+		var topic string
+		if t, exists := queryValues["topic"]; exists {
+			topic = t[0]
+			hashes := topicsCache[topic]
+			if hashes == nil {
+				makeErr(errors.New("no posts with that topic"))
 			}
-			commitN++
+			if len(hashes) > offset {
+				hashes = hashes[offset:]
+				commitN += offset
+			}
+			commits = &HashesIter{hashes}
+		} else {
+			for i := 0; i < offset; i++ {
+				if _, err = repoLog.Next(); err != nil {
+					break
+				}
+				commitN++
+			}
+			commits = repoLog
 		}
 
 		entries := make([]TableEntry, 0, PageEntries)
@@ -363,8 +407,9 @@ func serveHTTP() {
 			etagHash.Write([]byte(data))
 		}
 		etagHash.Write([]byte("INDEX"))
+		etagHash.Write([]byte(topic))
 		for i := 0; i < PageEntries; i++ {
-			commit, err := repoLog.Next()
+			commit, err := commits.Next()
 			if err != nil {
 				logEnded = true
 				break
@@ -372,9 +417,12 @@ func serveHTTP() {
 			etagHash.Write(commit.Hash[:])
 			commentsRaw := getNote(commentsTree, commit.Hash)
 			etagHash.Write(commentsRaw)
+			topicsRaw := getNote(topicsTree, commit.Hash)
+			etagHash.Write(topicsRaw)
 			entries = append(entries, TableEntry{
 				Commit:      commit,
 				CommentsRaw: commentsRaw,
+				TopicsRaw:   topicsRaw,
 			})
 		}
 		checkETag(etagHash)
@@ -393,13 +441,21 @@ func serveHTTP() {
 				entry.DomainURLs = append(entry.DomainURLs, makeA(line, u.Host))
 			}
 			entry.CommentsNum = len(parseComments(entry.CommentsRaw))
+			entry.Topics = parseTopics(entry.TopicsRaw)
 			entries[i] = entry
 		}
+		offsetPrev := offset - PageEntries
+		if offsetPrev < 0 {
+			offsetPrev = 0
+		}
 		tmpl := template.Must(template.New("index").Parse(TmplHTMLIndex))
 		os.Stdout.Write([]byte(startHeader(etagHash, gzipWriter != nil)))
 		err = tmpl.Execute(out, struct {
 			Version          string
 			Cfg              *Cfg
+			Topic            string
+			TopicsEnabled    bool
+			Topics           []string
 			CommentsEnabled  bool
 			AtomPostsFeed    string
 			AtomCommentsFeed string
@@ -411,11 +467,14 @@ func serveHTTP() {
 		}{
 			Version:          sgblog.Version,
 			Cfg:              cfg,
+			Topic:            topic,
+			TopicsEnabled:    topicsTree != nil,
+			Topics:           topicsCache.Topics(),
 			CommentsEnabled:  commentsTree != nil,
 			AtomPostsFeed:    AtomPostsFeed,
 			AtomCommentsFeed: AtomCommentsFeed,
 			Offset:           offset,
-			OffsetPrev:       offset - PageEntries,
+			OffsetPrev:       offsetPrev,
 			OffsetNext:       offset + PageEntries,
 			LogEnded:         logEnded,
 			Entries:          entries,
@@ -428,12 +487,32 @@ func serveHTTP() {
 		if err != nil {
 			makeErr(err)
 		}
+
+		var topic string
+		if t, exists := queryValues["topic"]; exists {
+			topic = t[0]
+		}
+
 		etagHash.Write([]byte("ATOM POSTS"))
+		etagHash.Write([]byte(topic))
 		etagHash.Write(commit.Hash[:])
 		checkETag(etagHash)
+		var title string
+		if topic == "" {
+			title = cfg.Title
+		} else {
+			title = fmt.Sprintf("%s (topic: %s)", cfg.Title, topic)
+		}
+		idHasher, err := blake2b.New256(nil)
+		if err != nil {
+			panic(err)
+		}
+		idHasher.Write([]byte("ATOM POSTS"))
+		idHasher.Write([]byte(cfg.AtomId))
+		idHasher.Write([]byte(topic))
 		feed := atom.Feed{
-			Title:   cfg.Title,
-			ID:      cfg.AtomId,
+			Title:   title,
+			ID:      "urn:uuid:" + bytes2uuid(idHasher.Sum(nil)),
 			Updated: atom.Time(commit.Author.When),
 			Link: []atom.Link{{
 				Rel:  "self",
@@ -441,16 +520,36 @@ func serveHTTP() {
 			}},
 			Author: &atom.Person{Name: cfg.AtomAuthor},
 		}
+
 		repoLog, err := repo.Log(&git.LogOptions{From: *headHash})
 		if err != nil {
 			makeErr(err)
 		}
+		var commits CommitIterNext
+		if topic == "" {
+			commits = repoLog
+		} else {
+			topicsCache, err := getTopicsCache(cfg, repoLog)
+			if err != nil {
+				makeErr(err)
+			}
+			hashes := topicsCache[topic]
+			if hashes == nil {
+				makeErr(errors.New("no posts with that topic"))
+			}
+			commits = &HashesIter{hashes}
+		}
+
 		for i := 0; i < PageEntries; i++ {
-			commit, err = repoLog.Next()
+			commit, err = commits.Next()
 			if err != nil {
 				break
 			}
 			lines := msgSplit(commit.Message)
+			var categories []atom.Category
+			for _, topic := range parseTopics(getNote(topicsTree, commit.Hash)) {
+				categories = append(categories, atom.Category{Term: topic})
+			}
 			feed.Entry = append(feed.Entry, &atom.Entry{
 				Title: lines[0],
 				ID:    "urn:uuid:" + bytes2uuid(commit.Hash[:]),
@@ -465,6 +564,7 @@ func serveHTTP() {
 					Type: "text",
 					Body: strings.Join(lines[2:], "\n"),
 				},
+				Category: categories,
 			})
 		}
 		data, err := xml.MarshalIndent(&feed, "", "  ")
@@ -577,6 +677,8 @@ func serveHTTP() {
 		}, "")
 		commentsRaw := getNote(commentsTree, commit.Hash)
 		etagHash.Write(commentsRaw)
+		topicsRaw := getNote(topicsTree, commit.Hash)
+		etagHash.Write(topicsRaw)
 		if strings.HasSuffix(pathInfo, AtomCommentsFeed) {
 			etagHash.Write([]byte("ATOM COMMENTS"))
 			checkETag(etagHash)
@@ -690,6 +792,7 @@ func serveHTTP() {
 			Lines           []string
 			NoteLines       []string
 			Comments        []CommentEntry
+			Topics          []string
 		}{
 			Version:         sgblog.Version,
 			Cfg:             cfg,
@@ -701,6 +804,7 @@ func serveHTTP() {
 			Lines:           lines[2:],
 			NoteLines:       notesLines,
 			Comments:        comments,
+			Topics:          parseTopics(topicsRaw),
 		})
 		if err != nil {
 			makeErr(err)
diff --git a/cmd/sgblog/main.go b/cmd/sgblog/main.go
index 68a3914..2e4a483 100644
--- a/cmd/sgblog/main.go
+++ b/cmd/sgblog/main.go
@@ -25,7 +25,9 @@ import (
 	"io/ioutil"
 	"os"
 	"regexp"
+	"sort"
 	"strings"
+	"text/scanner"
 
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/plumbing"
@@ -43,6 +45,8 @@ var (
 	notesTree    *object.Tree
 	commentsRef  *plumbing.Reference
 	commentsTree *object.Tree
+	topicsRef    *plumbing.Reference
+	topicsTree   *object.Tree
 )
 
 type Cfg struct {
@@ -64,6 +68,9 @@ type Cfg struct {
 	CommentsNotesRef string
 	CommentsEmail    string
 
+	TopicsNotesRef  string
+	TopicsCachePath string
+
 	GopherDomain string
 }
 
@@ -124,6 +131,17 @@ func parseComments(data []byte) []string {
 	return comments
 }
 
+func parseTopics(data []byte) []string {
+	var s scanner.Scanner
+	s.Init(bytes.NewBuffer(data))
+	topics := []string{}
+	for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
+		topics = append(topics, s.TokenText())
+	}
+	sort.Strings(topics)
+	return topics
+}
+
 func initRepo(cfg *Cfg) (*plumbing.Hash, error) {
 	var err error
 	repo, err = git.PlainOpen(cfg.GitPath)
@@ -143,6 +161,8 @@ func initRepo(cfg *Cfg) (*plumbing.Hash, error) {
 				notesRef = ref
 			case cfg.CommentsNotesRef:
 				commentsRef = ref
+			case cfg.TopicsNotesRef:
+				topicsRef = ref
 			}
 			return nil
 		})
@@ -156,6 +176,11 @@ func initRepo(cfg *Cfg) (*plumbing.Hash, error) {
 				commentsTree, _ = commentsCommit.Tree()
 			}
 		}
+		if topicsRef != nil {
+			if topicsCommit, err := repo.CommitObject(topicsRef.Hash()); err == nil {
+				topicsTree, _ = topicsCommit.Tree()
+			}
+		}
 	}
 	return &headHash, nil
 }
diff --git a/cmd/sgblog/topics.go b/cmd/sgblog/topics.go
new file mode 100644
index 0000000..907b3b3
--- /dev/null
+++ b/cmd/sgblog/topics.go
@@ -0,0 +1,131 @@
+/*
+SGBlog -- Git-backed CGI/inetd blogging/phlogging engine
+Copyright (C) 2020 Sergey Matveev 
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, version 3 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see .
+*/
+
+package main
+
+import (
+	"encoding/gob"
+	"io"
+	"os"
+	"sort"
+	"strconv"
+	"time"
+
+	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/go-git/go-git/v5/plumbing/object"
+)
+
+type TopicsCache map[string][]plumbing.Hash
+
+type TopicsCacheState struct {
+	Top   plumbing.Hash
+	Cache TopicsCache
+}
+
+func (tc TopicsCache) Topics() []string {
+	topics := make([]string, 0, len(tc))
+	for t, _ := range tc {
+		topics = append(topics, t)
+	}
+	sort.Strings(topics)
+	return topics
+}
+
+func getTopicsCache(cfg *Cfg, repoLog object.CommitIter) (TopicsCache, error) {
+	cache := TopicsCache(make(map[string][]plumbing.Hash))
+	if topicsTree == nil {
+		return cache, nil
+	}
+	top := topicsRef.Hash()
+
+	if cfg.TopicsCachePath != "" {
+		fd, err := os.Open(cfg.TopicsCachePath)
+		if err != nil {
+			goto NoCache
+		}
+		dec := gob.NewDecoder(fd)
+		var cacheState TopicsCacheState
+		err = dec.Decode(&cacheState)
+		fd.Close()
+		if err != nil {
+			goto NoCache
+		}
+		if cacheState.Top == top {
+			return cacheState.Cache, nil
+		}
+	}
+
+NoCache:
+	for {
+		commit, err := repoLog.Next()
+		if err != nil {
+			break
+		}
+		for _, topic := range parseTopics(getNote(topicsTree, commit.Hash)) {
+			cache[topic] = append(cache[topic], commit.Hash)
+		}
+	}
+
+	if cfg.TopicsCachePath != "" {
+		// Assume that probability of suffix collision is negligible
+		suffix := strconv.FormatInt(time.Now().UnixNano()+int64(os.Getpid()), 16)
+		tmpPath := cfg.TopicsCachePath + suffix
+		fd, err := os.OpenFile(
+			tmpPath,
+			os.O_RDWR|os.O_CREATE|os.O_EXCL,
+			os.FileMode(0666),
+		)
+		if err != nil {
+			return cache, err
+		}
+		enc := gob.NewEncoder(fd)
+		err = enc.Encode(&TopicsCacheState{top, cache})
+		if err != nil {
+			os.Remove(tmpPath)
+			fd.Close()
+			return cache, err
+		}
+		if err = fd.Sync(); err != nil {
+			os.Remove(tmpPath)
+			fd.Close()
+			return cache, err
+		}
+		if err = fd.Close(); err != nil {
+			os.Remove(tmpPath)
+			return cache, err
+		}
+		if err = os.Rename(tmpPath, cfg.TopicsCachePath); err != nil {
+			os.Remove(tmpPath)
+			return cache, err
+		}
+	}
+
+	return cache, nil
+}
+
+type HashesIter struct {
+	hashes []plumbing.Hash
+}
+
+func (s *HashesIter) Next() (*object.Commit, error) {
+	if len(s.hashes) == 0 {
+		return nil, io.EOF
+	}
+	var h plumbing.Hash
+	h, s.hashes = s.hashes[0], s.hashes[1:]
+	return repo.CommitObject(h)
+}
diff --git a/common.go b/common.go
index 5f80a47..6476388 100644
--- a/common.go
+++ b/common.go
@@ -2,6 +2,6 @@
 package sgblog
 
 const (
-	Version = "0.7.1"
+	Version = "0.8.0"
 	WhenFmt = "2006-01-02 15:04:05Z07:00"
 )
-- 
2.44.0