storage/sqlite/direct.go | 14 ++++++++++++-- storage/sqlite/sqlite-storage.go | 9 +++------ storage/sqlite/sqlite-storage_test.go | 11 +++-------- diff --git a/storage/sqlite/direct.go b/storage/sqlite/direct.go index 40c52383949c81e238064e44a4e25fa9ecd91c17..0287f56b2277e171f98014fa65ab4fa777a8e4ae 100644 --- a/storage/sqlite/direct.go +++ b/storage/sqlite/direct.go @@ -18,7 +18,7 @@ NewConnOpts InitDbOpts InitConnOpts GcBlobs bool - CacheBlobs bool + NoCacheBlobs bool BlobFlushInterval time.Duration } @@ -29,6 +29,11 @@ conn, err := newConn(opts.NewConnOpts) if err != nil { return } + if opts.PageSize == 0 { + // The largest size sqlite supports. I think we want this to be the smallest piece size we + // can expect, which is probably 1<<17. + opts.PageSize = 1 << 16 + } err = initDatabase(conn, opts.InitDbOpts) if err != nil { conn.Close() @@ -38,6 +43,11 @@ err = initConn(conn, opts.InitConnOpts) if err != nil { conn.Close() return + } + if opts.BlobFlushInterval == 0 && !opts.GcBlobs { + // This is influenced by typical busy timeouts, of 5-10s. We want to give other connections + // a few chances at getting a transaction through. + opts.BlobFlushInterval = time.Second } cl := &client{ conn: conn, @@ -153,7 +163,7 @@ create bool, ) (n int, err error) { p.l.Lock() defer p.l.Unlock() - if !p.opts.CacheBlobs { + if p.opts.NoCacheBlobs { defer p.forgetBlob() } blob, err := p.getBlob(create) diff --git a/storage/sqlite/sqlite-storage.go b/storage/sqlite/sqlite-storage.go index d1d890ee97ebf151725febab101b816367c1bdf2..a0450db7eba366c9fe8ca4bc7cdd6a9c69e09c1c 100644 --- a/storage/sqlite/sqlite-storage.go +++ b/storage/sqlite/sqlite-storage.go @@ -220,6 +220,9 @@ conns, err := NewPool(opts.NewPoolOpts) if err != nil { return } + if opts.PageSize == 0 { + opts.PageSize = 1 << 14 + } err = initPoolDatabase(conns, opts.InitDbOpts) if err != nil { conns.Close() @@ -336,12 +339,6 @@ } func initDatabase(conn conn, opts InitDbOpts) (err error) { if !opts.DontInitSchema { - if opts.PageSize == 0 { - // There doesn't seem to be an optimal size. I did try with the standard chunk size, but - // the difference is not convincing. - - opts.PageSize = 1 << 14 - } err = InitSchema(conn, opts.PageSize, !opts.NoTriggers) if err != nil { return diff --git a/storage/sqlite/sqlite-storage_test.go b/storage/sqlite/sqlite-storage_test.go index 45045df8fdb8939dc12d9622ef0429385cd84113..b19556b2f03d83fb70e543ccc019d2cc264786c6 100644 --- a/storage/sqlite/sqlite-storage_test.go +++ b/storage/sqlite/sqlite-storage_test.go @@ -86,9 +86,7 @@ c := qt.New(b) b.Run("CustomDirect", func(b *testing.B) { var opts NewDirectStorageOpts opts.Capacity = capacity - opts.BlobFlushInterval = time.Second - opts.CacheBlobs = true - opts.SetJournalMode = "off" + opts.NoTriggers = noTriggers benchOpts := func(b *testing.B) { opts.Path = filepath.Join(b.TempDir(), "storage.db") ci, err := NewDirectStorage(opts) @@ -96,9 +94,7 @@ c.Assert(err, qt.IsNil) defer ci.Close() runBench(b, ci) } - b.Run("Control", benchOpts) - opts.PageSize = 1 << 16 - b.Run("64KiB_PageSize", benchOpts) + b.Run("Default", benchOpts) }) for _, memory := range []bool{false, true} { b.Run(fmt.Sprintf("Memory=%v", memory), func(b *testing.B) { @@ -106,7 +102,6 @@ b.Run("Direct", func(b *testing.B) { var opts NewDirectStorageOpts opts.Memory = memory opts.Capacity = capacity - opts.CacheBlobs = true //opts.GcBlobs = true opts.BlobFlushInterval = time.Second opts.NoTriggers = noTriggers @@ -124,7 +119,7 @@ } for _, journalMode := range []string{"", "wal", "off", "truncate", "delete", "persist", "memory"} { opts.SetJournalMode = journalMode b.Run("JournalMode="+journalMode, func(b *testing.B) { - for _, mmapSize := range []int64{-1, 0, 1 << 23, 1 << 24, 1 << 25} { + for _, mmapSize := range []int64{-1} { if memory && mmapSize >= 0 { continue }