From: Matt Joiner <anacrolix@gmail.com>
Date: Thu, 6 May 2021 02:00:20 +0000 (+1000)
Subject: Benchmark different mmap sizes and journal modes
X-Git-Tag: v1.28.0~18^2~3
X-Git-Url: http://www.git.stargrave.org/?a=commitdiff_plain;h=abe003b6b34abf29688d06b9f11a57989de91167;p=btrtrc.git

Benchmark different mmap sizes and journal modes
---

diff --git a/storage/sqlite/direct.go b/storage/sqlite/direct.go
index 9da0f701..5f89ec98 100644
--- a/storage/sqlite/direct.go
+++ b/storage/sqlite/direct.go
@@ -13,6 +13,7 @@ import (
 type NewDirectStorageOpts struct {
 	NewConnOpts
 	InitDbOpts
+	InitConnOpts
 }
 
 // A convenience function that creates a connection pool, resource provider, and a pieces storage
@@ -22,16 +23,9 @@ func NewDirectStorage(opts NewDirectStorageOpts) (_ storage.ClientImplCloser, er
 	if err != nil {
 		return
 	}
-	journalMode := "delete"
-	if opts.Memory {
-		journalMode = "off"
-	}
-	err = initConn(conn, InitConnOpts{
-		SetJournalMode: journalMode,
-		MmapSizeOk:     true,
-		MmapSize:       1 << 25,
-	})
+	err = initConn(conn, opts.InitConnOpts)
 	if err != nil {
+		conn.Close()
 		return
 	}
 	err = initDatabase(conn, opts.InitDbOpts)
diff --git a/storage/sqlite/sqlite-storage.go b/storage/sqlite/sqlite-storage.go
index 8c569314..4192a58d 100644
--- a/storage/sqlite/sqlite-storage.go
+++ b/storage/sqlite/sqlite-storage.go
@@ -38,6 +38,8 @@ func (me InitConnOpts) JournalMode() string {
 	return "wal"
 }
 
+var UnexpectedJournalMode = errors.New("unexpected journal mode")
+
 func initConn(conn conn, opts InitConnOpts) error {
 	// Recursive triggers are required because we need to trim the blob_meta size after trimming to
 	// capacity. Hopefully we don't hit the recursion limit, and if we do, there's an error thrown.
@@ -53,7 +55,7 @@ func initConn(conn conn, opts InitConnOpts) error {
 		err = sqlitex.ExecTransient(conn, fmt.Sprintf(`pragma journal_mode=%s`, opts.SetJournalMode), func(stmt *sqlite.Stmt) error {
 			ret := stmt.ColumnText(0)
 			if ret != opts.SetJournalMode {
-				panic(ret)
+				return UnexpectedJournalMode
 			}
 			return nil
 		})
@@ -62,7 +64,10 @@ func initConn(conn conn, opts InitConnOpts) error {
 		}
 	}
 	if !opts.MmapSizeOk {
-		opts.MmapSize = 1 << 24 // 8 MiB
+		// Set the default. Currently it seems the library picks reasonable defaults, especially for
+		// wal.
+		opts.MmapSize = -1
+		//opts.MmapSize = 1 << 24 // 8 MiB
 	}
 	if opts.MmapSize >= 0 {
 		err = sqlitex.ExecTransient(conn, fmt.Sprintf(`pragma mmap_size=%d`, opts.MmapSize), nil)
diff --git a/storage/sqlite/sqlite-storage_test.go b/storage/sqlite/sqlite-storage_test.go
index f2e7ad29..633d4358 100644
--- a/storage/sqlite/sqlite-storage_test.go
+++ b/storage/sqlite/sqlite-storage_test.go
@@ -2,6 +2,7 @@ package sqliteStorage
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -12,6 +13,7 @@ import (
 	_ "github.com/anacrolix/envpprof"
 	"github.com/anacrolix/torrent/storage"
 	test_storage "github.com/anacrolix/torrent/storage/test"
+	"github.com/dustin/go-humanize"
 	qt "github.com/frankban/quicktest"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -82,10 +84,36 @@ func BenchmarkMarkComplete(b *testing.B) {
 				opts.Memory = memory
 				opts.Path = filepath.Join(b.TempDir(), "storage.db")
 				opts.Capacity = capacity
-				ci, err := NewDirectStorage(opts)
-				c.Assert(err, qt.IsNil)
-				defer ci.Close()
-				runBench(b, ci)
+				directBench := func(b *testing.B) {
+					ci, err := NewDirectStorage(opts)
+					if errors.Is(err, UnexpectedJournalMode) {
+						b.Skipf("setting journal mode %q: %v", opts.SetJournalMode, err)
+					}
+					c.Assert(err, qt.IsNil)
+					defer ci.Close()
+					runBench(b, ci)
+				}
+				for _, journalMode := range []string{"", "wal", "off", "delete", "memory"} {
+					opts.SetJournalMode = journalMode
+					b.Run("JournalMode="+journalMode, func(b *testing.B) {
+						for _, mmapSize := range []int64{-1, 0, 1 << 24, 1 << 25, 1 << 26} {
+							if memory && mmapSize >= 0 {
+								continue
+							}
+							b.Run(fmt.Sprintf("MmapSize=%s", func() string {
+								if mmapSize < 0 {
+									return "default"
+								} else {
+									return humanize.IBytes(uint64(mmapSize))
+								}
+							}()), func(b *testing.B) {
+								opts.MmapSize = mmapSize
+								opts.MmapSizeOk = true
+								directBench(b)
+							})
+						}
+					})
+				}
 			})
 			b.Run("ResourcePieces", func(b *testing.B) {
 				for _, batchWrites := range []bool{false, true} {