Add possum storage

diff --git a/go.mod b/go.mod
index 1b0f5816fee11a28622fffb7e94d20305efc2bf7..529149f9ce559dcf7a5886ebcae410738fdd270c 100644 (file)
--- a/go.mod
+++ b/go.mod
@@ -1,6 +1,8 @@
 module github.com/anacrolix/torrent
 
-go 1.20
+go 1.21.4
+
+toolchain go1.21.7
 
 require (
        github.com/RoaringBitmap/roaring v1.2.3
@@ -11,13 +13,14 @@ require (
        github.com/anacrolix/dht/v2 v2.19.2-0.20221121215055-066ad8494444
        github.com/anacrolix/envpprof v1.3.0
        github.com/anacrolix/fuse v0.2.0
-       github.com/anacrolix/generics v0.0.0-20230816105729-c755655aee45
+       github.com/anacrolix/generics v0.0.0-20230911070922-5dd7545c6b13
        github.com/anacrolix/go-libutp v1.3.1
        github.com/anacrolix/log v0.14.6-0.20231202035202-ed7a02cad0b4
        github.com/anacrolix/missinggo v1.3.0
        github.com/anacrolix/missinggo/perf v1.0.0
-       github.com/anacrolix/missinggo/v2 v2.7.2-0.20230527121029-a582b4f397b9
+       github.com/anacrolix/missinggo/v2 v2.7.3
        github.com/anacrolix/multiless v0.3.0
+       github.com/anacrolix/possum/go v0.0.0-20240222034319-2fe0737d4315
        github.com/anacrolix/squirrel v0.6.0
        github.com/anacrolix/sync v0.5.1
        github.com/anacrolix/tagflag v1.3.0

diff --git a/go.sum b/go.sum
index c8ce30f99d6495b690a4055859456b85ea883c53..934f9de8ecc34e0f6da2ab9d1481981a952ea196 100644 (file)
--- a/go.sum
+++ b/go.sum
@@ -34,9 +34,11 @@ crawshaw.io/iox v0.0.0-20181124134642-c51c3df30797/go.mod h1:sXBiorCo8c46JlQV3oX
 crawshaw.io/sqlite v0.3.2/go.mod h1:igAO5JulrQ1DbdZdtVq48mnZUBAPOeFzer7VhDWNtW4=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 filippo.io/edwards25519 v1.0.0-rc.1 h1:m0VOOB23frXZvAOK44usCgLWvtsxIoMCTBGJZlpmGfU=
+filippo.io/edwards25519 v1.0.0-rc.1/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7eHn5EwJns=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74=
+github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/RoaringBitmap/roaring v0.4.7/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJx2zgh7AcNke4w=
 github.com/RoaringBitmap/roaring v0.4.17/go.mod h1:D3qVegWTmfCaX4Bl5CrBE9hfrSrrXIr8KVNvRsDi1NI=
@@ -48,9 +50,11 @@ github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMx
 github.com/ajwerner/btree v0.0.0-20211221152037-f427b3e689c0 h1:byYvvbfSo3+9efR4IeReh77gVs4PnNDR3AMOE9NJ7a0=
 github.com/ajwerner/btree v0.0.0-20211221152037-f427b3e689c0/go.mod h1:q37NoqncT41qKc048STsifIt69LfUJ8SrWWcz/yam5k=
 github.com/alecthomas/assert/v2 v2.0.0-alpha3 h1:pcHeMvQ3OMstAWgaeaXIAL8uzB9xMm2zlxt+/4ml8lk=
+github.com/alecthomas/assert/v2 v2.0.0-alpha3/go.mod h1:+zD0lmDXTeQj7TgDgCt0ePWxb0hMC1G+PGTsTCv1B9o=
 github.com/alecthomas/atomic v0.1.0-alpha2 h1:dqwXmax66gXvHhsOS4pGPZKqYOlTkapELkLb3MNdlH8=
 github.com/alecthomas/atomic v0.1.0-alpha2/go.mod h1:zD6QGEyw49HIq19caJDc2NMXAy8rNi9ROrxtMXATfyI=
 github.com/alecthomas/repr v0.0.0-20210801044451-80ca428c5142 h1:8Uy0oSf5co/NZXje7U1z8Mpep++QJOldL2hs/sBQf48=
+github.com/alecthomas/repr v0.0.0-20210801044451-80ca428c5142/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygvNfaDQL8=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
@@ -73,8 +77,8 @@ github.com/anacrolix/envpprof v1.3.0 h1:WJt9bpuT7A/CDCxPOv/eeZqHWlle/Y0keJUvc6tc
 github.com/anacrolix/envpprof v1.3.0/go.mod h1:7QIG4CaX1uexQ3tqd5+BRa/9e2D02Wcertl6Yh0jCB0=
 github.com/anacrolix/fuse v0.2.0 h1:pc+To78kI2d/WUjIyrsdqeJQAesuwpGxlI3h1nAv3Do=
 github.com/anacrolix/fuse v0.2.0/go.mod h1:Kfu02xBwnySDpH3N23BmrP3MDfwAQGRLUCj6XyeOvBQ=
-github.com/anacrolix/generics v0.0.0-20230816105729-c755655aee45 h1:Kmcl3I9K2+5AdnnR7hvrnVT0TLeFWWMa9bxnm55aVIg=
-github.com/anacrolix/generics v0.0.0-20230816105729-c755655aee45/go.mod h1:ff2rHB/joTV03aMSSn/AZNnaIpUw0h3njetGsaXcMy8=
+github.com/anacrolix/generics v0.0.0-20230911070922-5dd7545c6b13 h1:qwOprPTDMM3BASJRf84mmZnTXRsPGGJ8xoHKQS7m3so=
+github.com/anacrolix/generics v0.0.0-20230911070922-5dd7545c6b13/go.mod h1:ff2rHB/joTV03aMSSn/AZNnaIpUw0h3njetGsaXcMy8=
 github.com/anacrolix/go-libutp v1.3.1 h1:idJzreNLl+hNjGC3ZnUOjujEaryeOGgkwHLqSGoige0=
 github.com/anacrolix/go-libutp v1.3.1/go.mod h1:heF41EC8kN0qCLMokLBVkB8NXiLwx3t8R8810MTNI5o=
 github.com/anacrolix/log v0.3.0/go.mod h1:lWvLTqzAnCWPJA08T2HCstZi0L1y2Wyvm3FJgwU9jwU=
@@ -96,13 +100,15 @@ github.com/anacrolix/missinggo/perf v1.0.0/go.mod h1:ljAFWkBuzkO12MQclXzZrosP5ur
 github.com/anacrolix/missinggo/v2 v2.2.0/go.mod h1:o0jgJoYOyaoYQ4E2ZMISVa9c88BbUBVQQW4QeRkNCGY=
 github.com/anacrolix/missinggo/v2 v2.5.1/go.mod h1:WEjqh2rmKECd0t1VhQkLGTdIWXO6f6NLjp5GlMZ+6FA=
 github.com/anacrolix/missinggo/v2 v2.5.2/go.mod h1:yNvsLrtZYRYCOI+KRH/JM8TodHjtIE/bjOGhQaLOWIE=
-github.com/anacrolix/missinggo/v2 v2.7.2-0.20230527121029-a582b4f397b9 h1:W/oGeHhYwxueeiDjQfmK9G+X9M2xJgfTtow62v0TWAs=
-github.com/anacrolix/missinggo/v2 v2.7.2-0.20230527121029-a582b4f397b9/go.mod h1:mIEtp9pgaXqt8VQ3NQxFOod/eQ1H0D1XsZzKUQfwtac=
+github.com/anacrolix/missinggo/v2 v2.7.3 h1:Ee//CmZBMadeNiYB/hHo9ly2PFOEZ4Fhsbnug3rDAIE=
+github.com/anacrolix/missinggo/v2 v2.7.3/go.mod h1:mIEtp9pgaXqt8VQ3NQxFOod/eQ1H0D1XsZzKUQfwtac=
 github.com/anacrolix/mmsg v0.0.0-20180515031531-a4a3ba1fc8bb/go.mod h1:x2/ErsYUmT77kezS63+wzZp8E3byYB0gzirM/WMBLfw=
 github.com/anacrolix/mmsg v1.0.0 h1:btC7YLjOn29aTUAExJiVUhQOuf/8rhm+/nWCMAnL3Hg=
 github.com/anacrolix/mmsg v1.0.0/go.mod h1:x8kRaJY/dCrY9Al0PEcj1mb/uFHwP6GCJ9fLl4thEPc=
 github.com/anacrolix/multiless v0.3.0 h1:5Bu0DZncjE4e06b9r1Ap2tUY4Au0NToBP5RpuEngSis=
 github.com/anacrolix/multiless v0.3.0/go.mod h1:TrCLEZfIDbMVfLoQt5tOoiBS/uq4y8+ojuEVVvTNPX4=
+github.com/anacrolix/possum/go v0.0.0-20240222034319-2fe0737d4315 h1:bwdpi8LWvEHrNgsu44SKmCzLeZ5iEH4rwmNafDdRHJ4=
+github.com/anacrolix/possum/go v0.0.0-20240222034319-2fe0737d4315/go.mod h1:pw5HEMBSiL+otYzHe4q5jGaVuy5unl+Mt4Bx6SDemW8=
 github.com/anacrolix/squirrel v0.6.0 h1:ovfWW42wcGzrVYYI9s56pEYzfeTwtXxCCvSd+KwvUEA=
 github.com/anacrolix/squirrel v0.6.0/go.mod h1:60vdNPUbK1jYWePp39Wqn9whHm12Yb9JEuwOXzLMDuY=
 github.com/anacrolix/stm v0.2.0/go.mod h1:zoVQRvSiGjGoTmbM0vSLIiaKjWtNPeTvXUSdJQA4hsg=
@@ -301,6 +307,7 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4Zs
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
+github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo=
 github.com/huandu/xstrings v1.2.0/go.mod h1:DvyZB1rfVYsBIigL8HwpZgxHwXozlTgGqn63UyNX5k4=
@@ -469,6 +476,7 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1
 github.com/smartystreets/assertions v0.0.0-20190215210624-980c5ac6f3ac/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
 github.com/smartystreets/goconvey v0.0.0-20190306220146-200a235640ff/go.mod h1:KSQcGKpxUMHk3nbYzs/tIBAM2iDooCn0BmttHOJEbLs=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/stephens2424/writerset v1.0.2/go.mod h1:aS2JhsMn6eA7e82oNmW4rfsgAOp9COBTTl8mzkwADnc=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -525,6 +533,7 @@ go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqe
 go.opentelemetry.io/proto/otlp v0.18.0 h1:W5hyXNComRa23tGpKwG+FRAc4rfF6ZUg1JReK+QHS80=
 go.opentelemetry.io/proto/otlp v0.18.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
 go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA=
+go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=

diff --git a/storage/piece-resource.go b/storage/piece-resource.go
index 5327f31d1b10e5ca08b1ffeb03e4e76af8ca39c4..a891e9d246af0e0b94dc610e87229830ca47a424 100644 (file)
--- a/storage/piece-resource.go
+++ b/storage/piece-resource.go
@@ -3,11 +3,11 @@ package storage
 import (
        "bytes"
        "fmt"
+       "github.com/anacrolix/sync"
        "io"
        "path"
        "sort"
        "strconv"
-       "sync"
 
        "github.com/anacrolix/missinggo/v2/resource"
 

diff --git a/storage/possum/possum-provider.go b/storage/possum/possum-provider.go
new file mode 100644 (file)
index 0000000..e67cdad
--- /dev/null
+++ b/storage/possum/possum-provider.go
@@ -0,0 +1,135 @@
+//go:build !android
+
+package possumTorrentStorage
+
+import (
+       "cmp"
+       "fmt"
+       "github.com/anacrolix/log"
+       possum "github.com/anacrolix/possum/go"
+       possumResource "github.com/anacrolix/possum/go/resource"
+       "github.com/anacrolix/torrent/storage"
+       "io"
+       "sort"
+       "strconv"
+)
+
+// Extends possum resource.Provider with an efficient implementation of torrent
+// storage.ConsecutiveChunkReader.
+type Provider struct {
+       possumResource.Provider
+       Logger log.Logger
+}
+
+var _ storage.ConsecutiveChunkReader = Provider{}
+
+// Sorts by a precomputed key but swaps on another slice at the same time.
+type keySorter[T any, K cmp.Ordered] struct {
+       orig []T
+       keys []K
+}
+
+func (o keySorter[T, K]) Len() int {
+       return len(o.keys)
+}
+
+func (o keySorter[T, K]) Less(i, j int) bool {
+       return o.keys[i] < o.keys[j]
+}
+
+func (o keySorter[T, K]) Swap(i, j int) {
+       o.keys[i], o.keys[j] = o.keys[j], o.keys[i]
+       o.orig[i], o.orig[j] = o.orig[j], o.orig[i]
+}
+
+// TODO: Should the parent ReadConsecutiveChunks method take the expected number of bytes to avoid
+// trying to read discontinuous or incomplete sequences of chunks?
+func (p Provider) ReadConsecutiveChunks(prefix string) (rc io.ReadCloser, err error) {
+       p.Logger.Levelf(log.Debug, "ReadConsecutiveChunks(%q)", prefix)
+       //debug.PrintStack()
+       pr, err := p.Handle.NewReader()
+       if err != nil {
+               return
+       }
+       defer func() {
+               if err != nil {
+                       pr.End()
+               }
+       }()
+       items, err := pr.ListItems(prefix)
+       if err != nil {
+               return
+       }
+       keys := make([]int64, 0, len(items))
+       for _, item := range items {
+               var i int64
+               offsetStr := item.Key
+               i, err = strconv.ParseInt(offsetStr, 10, 64)
+               if err != nil {
+                       err = fmt.Errorf("failed to parse offset %q: %w", offsetStr, err)
+                       return
+               }
+               keys = append(keys, i)
+       }
+       sort.Sort(keySorter[possum.Item, int64]{items, keys})
+       offset := int64(0)
+       consValues := make([]consecutiveValue, 0, len(items))
+       for i, item := range items {
+               itemOffset := keys[i]
+               if itemOffset > offset {
+                       // We can't provide a continuous read.
+                       break
+               }
+               if itemOffset+item.Stat.Size() <= offset {
+                       // This item isn't needed
+                       continue
+               }
+               var v possum.Value
+               v, err = pr.Add(prefix + item.Key)
+               if err != nil {
+                       return
+               }
+               consValues = append(consValues, consecutiveValue{
+                       pv:     v,
+                       offset: itemOffset,
+                       size:   item.Stat.Size(),
+               })
+               offset += item.Stat.Size() - (offset - itemOffset)
+       }
+       err = pr.Begin()
+       if err != nil {
+               return
+       }
+       rc, pw := io.Pipe()
+       go func() {
+               defer pr.End()
+               err := p.writeConsecutiveValues(consValues, pw)
+               err = pw.CloseWithError(err)
+               if err != nil {
+                       panic(err)
+               }
+       }()
+       return
+}
+
+type consecutiveValue struct {
+       pv     possum.Value
+       offset int64
+       size   int64
+}
+
+func (pp Provider) writeConsecutiveValues(
+       values []consecutiveValue, pw *io.PipeWriter,
+) (err error) {
+       off := int64(0)
+       for _, v := range values {
+               var n int64
+               valueOff := off - v.offset
+               n, err = io.Copy(pw, io.NewSectionReader(v.pv, valueOff, v.size-valueOff))
+               if err != nil {
+                       return
+               }
+               off += n
+       }
+       return nil
+}

diff --git a/storage/possum/possum_test.go b/storage/possum/possum_test.go
new file mode 100644 (file)
index 0000000..6bd350b
--- /dev/null
+++ b/storage/possum/possum_test.go
@@ -0,0 +1,30 @@
+package possumTorrentStorage
+
+import (
+       g "github.com/anacrolix/generics"
+       "github.com/anacrolix/log"
+       possum "github.com/anacrolix/possum/go"
+       possumResource "github.com/anacrolix/possum/go/resource"
+       "github.com/anacrolix/torrent/storage"
+       test_storage "github.com/anacrolix/torrent/storage/test"
+       "testing"
+)
+
+// This should be made to mirror the benchmarks for sqlite storage.
+func BenchmarkProvider(b *testing.B) {
+       possumDir, err := possum.Open(b.TempDir())
+       if err != nil {
+               b.Fatal(err)
+       }
+       possumDir.SetInstanceLimits(possum.Limits{
+               DisableHolePunching: false,
+               MaxValueLengthSum:   g.Some[uint64](test_storage.DefaultPieceSize * test_storage.DefaultNumPieces / 2),
+       })
+       defer possumDir.Close()
+       possumProvider := possumResource.Provider{Handle: possumDir}
+       possumTorrentProvider := Provider{Provider: possumProvider, Logger: log.Default}
+       clientStorageImpl := storage.NewResourcePiecesOpts(
+               possumTorrentProvider,
+               storage.ResourcePiecesOpts{LeaveIncompleteChunks: true})
+       test_storage.BenchmarkPieceMarkComplete(b, clientStorageImpl, test_storage.DefaultPieceSize, test_storage.DefaultNumPieces, 0)
+}