From: Sergey Matveev <stargrave@stargrave.org>
Date: Fri, 11 Mar 2022 14:56:46 +0000 (+0300)
Subject: Less gob calls -- several times faster search
X-Git-Url: http://www.git.stargrave.org/?p=rutrackerer.git;a=commitdiff_plain;h=3be50f5d9fbecbd1c1d3d8c59cd3649e3ffb06da

Less gob calls -- several times faster search
---

diff --git a/README b/README
index fae6d14..38341a5 100644
--- a/README
+++ b/README
@@ -16,9 +16,9 @@ quickly feed in further commands. That XML is huge, so you can get some
 troubles searching in it quickly. So here are the indexer, that extracts
 titles, sizes, hashes and XML offsets:
 
-    $ cmd/index < rutracker-XXX.xml > rutracker.gob
+    $ cmd/index < rutracker-XXX.xml | cmd/repack > rutracker.gob
 
-On my computer is works for several minutes, that is pretty fast.
+On my computer that takes seven minutes.
 
 After that, you can search the desired title like that:
 
diff --git a/cmd/repack/main.go b/cmd/repack/main.go
new file mode 100644
index 0000000..61f99dc
--- /dev/null
+++ b/cmd/repack/main.go
@@ -0,0 +1,30 @@
+package main
+
+import (
+	"bufio"
+	"encoding/gob"
+	"io"
+	"os"
+
+	"go.stargrave.org/rutrackerer"
+)
+
+func main() {
+	gobDec := gob.NewDecoder(bufio.NewReader(os.Stdin))
+	torrents := make([]*rutrackerer.Torrent, 0, 1<<20)
+	var err error
+	for {
+		var torrent rutrackerer.Torrent
+		if err = gobDec.Decode(&torrent); err != nil {
+			if err == io.EOF {
+				break
+			}
+			panic(err)
+		}
+		torrents = append(torrents, &torrent)
+	}
+	gobEnc := gob.NewEncoder(os.Stdout)
+	if err = gobEnc.Encode(torrents); err != nil {
+		panic(err)
+	}
+}
diff --git a/cmd/search/main.go b/cmd/search/main.go
index 0645be0..ff757b6 100644
--- a/cmd/search/main.go
+++ b/cmd/search/main.go
@@ -5,7 +5,6 @@ import (
 	"encoding/gob"
 	"encoding/hex"
 	"fmt"
-	"io"
 	"os"
 	"os/exec"
 	"strconv"
@@ -28,7 +27,11 @@ func main() {
 	if err = cmd.Start(); err != nil {
 		panic(err)
 	}
-	torrents := make([]*rutrackerer.Torrent, 0, 1<<20)
+	var torrents []*rutrackerer.Torrent
+	gobDec := gob.NewDecoder(bufio.NewReader(os.Stdin))
+	if err = gobDec.Decode(&torrents); err != nil {
+		panic(err)
+	}
 	printer := make(chan struct{})
 	go func() {
 		scanner := bufio.NewScanner(grepStdout)
@@ -63,17 +66,8 @@ func main() {
 		}
 		close(printer)
 	}()
-	gobDec := gob.NewDecoder(bufio.NewReader(os.Stdin))
-	for {
-		var torrent rutrackerer.Torrent
-		if err = gobDec.Decode(&torrent); err != nil {
-			if err == io.EOF {
-				break
-			}
-			panic(err)
-		}
-		torrents = append(torrents, &torrent)
-		if _, err = grepStdin.Write([]byte(torrent.Title + "\n")); err != nil {
+	for _, t := range torrents {
+		if _, err = grepStdin.Write([]byte(t.Title + "\n")); err != nil {
 			panic(err)
 		}
 	}