Less gob calls -- several times faster search

diff --git a/README b/README
index fae6d1450d07298ac829b0996232e9ce899daafe..38341a576e496f6feebbcde4182576ef292f7d07 100644 (file)
--- a/README
+++ b/README
@@ -16,9 +16,9 @@ quickly feed in further commands. That XML is huge, so you can get some
 troubles searching in it quickly. So here are the indexer, that extracts
 titles, sizes, hashes and XML offsets:
 
-    $ cmd/index < rutracker-XXX.xml > rutracker.gob
+    $ cmd/index < rutracker-XXX.xml | cmd/repack > rutracker.gob
 
-On my computer is works for several minutes, that is pretty fast.
+On my computer that takes seven minutes.
 
 After that, you can search the desired title like that:
 

diff --git a/cmd/repack/main.go b/cmd/repack/main.go
new file mode 100644 (file)
index 0000000..61f99dc
--- /dev/null
+++ b/cmd/repack/main.go
@@ -0,0 +1,30 @@
+package main
+
+import (
+       "bufio"
+       "encoding/gob"
+       "io"
+       "os"
+
+       "go.stargrave.org/rutrackerer"
+)
+
+func main() {
+       gobDec := gob.NewDecoder(bufio.NewReader(os.Stdin))
+       torrents := make([]*rutrackerer.Torrent, 0, 1<<20)
+       var err error
+       for {
+               var torrent rutrackerer.Torrent
+               if err = gobDec.Decode(&torrent); err != nil {
+                       if err == io.EOF {
+                               break
+                       }
+                       panic(err)
+               }
+               torrents = append(torrents, &torrent)
+       }
+       gobEnc := gob.NewEncoder(os.Stdout)
+       if err = gobEnc.Encode(torrents); err != nil {
+               panic(err)
+       }
+}

diff --git a/cmd/search/main.go b/cmd/search/main.go
index 0645be046ad8482c2e95e050de78c81a9f40de2c..ff757b607511f5aacd409258fc52978f232cb21e 100644 (file)
--- a/cmd/search/main.go
+++ b/cmd/search/main.go
@@ -5,7 +5,6 @@ import (
        "encoding/gob"
        "encoding/hex"
        "fmt"
-       "io"
        "os"
        "os/exec"
        "strconv"
@@ -28,7 +27,11 @@ func main() {
        if err = cmd.Start(); err != nil {
                panic(err)
        }
-       torrents := make([]*rutrackerer.Torrent, 0, 1<<20)
+       var torrents []*rutrackerer.Torrent
+       gobDec := gob.NewDecoder(bufio.NewReader(os.Stdin))
+       if err = gobDec.Decode(&torrents); err != nil {
+               panic(err)
+       }
        printer := make(chan struct{})
        go func() {
                scanner := bufio.NewScanner(grepStdout)
@@ -63,17 +66,8 @@ func main() {
                }
                close(printer)
        }()
-       gobDec := gob.NewDecoder(bufio.NewReader(os.Stdin))
-       for {
-               var torrent rutrackerer.Torrent
-               if err = gobDec.Decode(&torrent); err != nil {
-                       if err == io.EOF {
-                               break
-                       }
-                       panic(err)
-               }
-               torrents = append(torrents, &torrent)
-               if _, err = grepStdin.Write([]byte(torrent.Title + "\n")); err != nil {
+       for _, t := range torrents {
+               if _, err = grepStdin.Write([]byte(t.Title + "\n")); err != nil {
                        panic(err)
                }
        }