Set smarter defaults

diff --git a/storage/sqlite/direct.go b/storage/sqlite/direct.go
index 40c52383949c81e238064e44a4e25fa9ecd91c17..0287f56b2277e171f98014fa65ab4fa777a8e4ae 100644 (file)
--- a/storage/sqlite/direct.go
+++ b/storage/sqlite/direct.go
@@ -18,7 +18,7 @@ type NewDirectStorageOpts struct {
        InitDbOpts
        InitConnOpts
        GcBlobs           bool
-       CacheBlobs        bool
+       NoCacheBlobs      bool
        BlobFlushInterval time.Duration
 }
 
@@ -29,6 +29,11 @@ func NewDirectStorage(opts NewDirectStorageOpts) (_ storage.ClientImplCloser, er
        if err != nil {
                return
        }
+       if opts.PageSize == 0 {
+               // The largest size sqlite supports. I think we want this to be the smallest piece size we
+               // can expect, which is probably 1<<17.
+               opts.PageSize = 1 << 16
+       }
        err = initDatabase(conn, opts.InitDbOpts)
        if err != nil {
                conn.Close()
@@ -39,6 +44,11 @@ func NewDirectStorage(opts NewDirectStorageOpts) (_ storage.ClientImplCloser, er
                conn.Close()
                return
        }
+       if opts.BlobFlushInterval == 0 && !opts.GcBlobs {
+               // This is influenced by typical busy timeouts, of 5-10s. We want to give other connections
+               // a few chances at getting a transaction through.
+               opts.BlobFlushInterval = time.Second
+       }
        cl := &client{
                conn:  conn,
                blobs: make(map[string]*sqlite.Blob),
@@ -153,7 +163,7 @@ func (p piece) doAtIoWithBlob(
 ) (n int, err error) {
        p.l.Lock()
        defer p.l.Unlock()
-       if !p.opts.CacheBlobs {
+       if p.opts.NoCacheBlobs {
                defer p.forgetBlob()
        }
        blob, err := p.getBlob(create)

diff --git a/storage/sqlite/sqlite-storage.go b/storage/sqlite/sqlite-storage.go
index d1d890ee97ebf151725febab101b816367c1bdf2..a0450db7eba366c9fe8ca4bc7cdd6a9c69e09c1c 100644 (file)
--- a/storage/sqlite/sqlite-storage.go
+++ b/storage/sqlite/sqlite-storage.go
@@ -220,6 +220,9 @@ func NewPiecesStorage(opts NewPiecesStorageOpts) (_ storage.ClientImplCloser, er
        if err != nil {
                return
        }
+       if opts.PageSize == 0 {
+               opts.PageSize = 1 << 14
+       }
        err = initPoolDatabase(conns, opts.InitDbOpts)
        if err != nil {
                conns.Close()
@@ -336,12 +339,6 @@ func newOpenUri(opts NewConnOpts) string {
 
 func initDatabase(conn conn, opts InitDbOpts) (err error) {
        if !opts.DontInitSchema {
-               if opts.PageSize == 0 {
-                       // There doesn't seem to be an optimal size. I did try with the standard chunk size, but
-                       // the difference is not convincing.
-
-                       opts.PageSize = 1 << 14
-               }
                err = InitSchema(conn, opts.PageSize, !opts.NoTriggers)
                if err != nil {
                        return

diff --git a/storage/sqlite/sqlite-storage_test.go b/storage/sqlite/sqlite-storage_test.go
index 45045df8fdb8939dc12d9622ef0429385cd84113..b19556b2f03d83fb70e543ccc019d2cc264786c6 100644 (file)
--- a/storage/sqlite/sqlite-storage_test.go
+++ b/storage/sqlite/sqlite-storage_test.go
@@ -86,9 +86,7 @@ func BenchmarkMarkComplete(b *testing.B) {
        b.Run("CustomDirect", func(b *testing.B) {
                var opts NewDirectStorageOpts
                opts.Capacity = capacity
-               opts.BlobFlushInterval = time.Second
-               opts.CacheBlobs = true
-               opts.SetJournalMode = "off"
+               opts.NoTriggers = noTriggers
                benchOpts := func(b *testing.B) {
                        opts.Path = filepath.Join(b.TempDir(), "storage.db")
                        ci, err := NewDirectStorage(opts)
@@ -96,9 +94,7 @@ func BenchmarkMarkComplete(b *testing.B) {
                        defer ci.Close()
                        runBench(b, ci)
                }
-               b.Run("Control", benchOpts)
-               opts.PageSize = 1 << 16
-               b.Run("64KiB_PageSize", benchOpts)
+               b.Run("Default", benchOpts)
        })
        for _, memory := range []bool{false, true} {
                b.Run(fmt.Sprintf("Memory=%v", memory), func(b *testing.B) {
@@ -106,7 +102,6 @@ func BenchmarkMarkComplete(b *testing.B) {
                                var opts NewDirectStorageOpts
                                opts.Memory = memory
                                opts.Capacity = capacity
-                               opts.CacheBlobs = true
                                //opts.GcBlobs = true
                                opts.BlobFlushInterval = time.Second
                                opts.NoTriggers = noTriggers
@@ -124,7 +119,7 @@ func BenchmarkMarkComplete(b *testing.B) {
                                for _, journalMode := range []string{"", "wal", "off", "truncate", "delete", "persist", "memory"} {
                                        opts.SetJournalMode = journalMode
                                        b.Run("JournalMode="+journalMode, func(b *testing.B) {
-                                               for _, mmapSize := range []int64{-1, 0, 1 << 23, 1 << 24, 1 << 25} {
+                                               for _, mmapSize := range []int64{-1} {
                                                        if memory && mmapSize >= 0 {
                                                                continue
                                                        }