From: Matt Joiner Date: Thu, 22 Feb 2024 03:49:03 +0000 (+1100) Subject: Add possum storage X-Git-Url: http://www.git.stargrave.org/?a=commitdiff_plain;h=47267688ac6730ae093e07e24ef4ca11d8a3230a;p=btrtrc.git Add possum storage --- diff --git a/go.mod b/go.mod index 1b0f5816..529149f9 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/anacrolix/torrent -go 1.20 +go 1.21.4 + +toolchain go1.21.7 require ( github.com/RoaringBitmap/roaring v1.2.3 @@ -11,13 +13,14 @@ require ( github.com/anacrolix/dht/v2 v2.19.2-0.20221121215055-066ad8494444 github.com/anacrolix/envpprof v1.3.0 github.com/anacrolix/fuse v0.2.0 - github.com/anacrolix/generics v0.0.0-20230816105729-c755655aee45 + github.com/anacrolix/generics v0.0.0-20230911070922-5dd7545c6b13 github.com/anacrolix/go-libutp v1.3.1 github.com/anacrolix/log v0.14.6-0.20231202035202-ed7a02cad0b4 github.com/anacrolix/missinggo v1.3.0 github.com/anacrolix/missinggo/perf v1.0.0 - github.com/anacrolix/missinggo/v2 v2.7.2-0.20230527121029-a582b4f397b9 + github.com/anacrolix/missinggo/v2 v2.7.3 github.com/anacrolix/multiless v0.3.0 + github.com/anacrolix/possum/go v0.0.0-20240222034319-2fe0737d4315 github.com/anacrolix/squirrel v0.6.0 github.com/anacrolix/sync v0.5.1 github.com/anacrolix/tagflag v1.3.0 diff --git a/go.sum b/go.sum index c8ce30f9..934f9de8 100644 --- a/go.sum +++ b/go.sum @@ -34,9 +34,11 @@ crawshaw.io/iox v0.0.0-20181124134642-c51c3df30797/go.mod h1:sXBiorCo8c46JlQV3oX crawshaw.io/sqlite v0.3.2/go.mod h1:igAO5JulrQ1DbdZdtVq48mnZUBAPOeFzer7VhDWNtW4= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= filippo.io/edwards25519 v1.0.0-rc.1 h1:m0VOOB23frXZvAOK44usCgLWvtsxIoMCTBGJZlpmGfU= +filippo.io/edwards25519 v1.0.0-rc.1/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7eHn5EwJns= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/RoaringBitmap/roaring v0.4.7/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJx2zgh7AcNke4w= github.com/RoaringBitmap/roaring v0.4.17/go.mod h1:D3qVegWTmfCaX4Bl5CrBE9hfrSrrXIr8KVNvRsDi1NI= @@ -48,9 +50,11 @@ github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMx github.com/ajwerner/btree v0.0.0-20211221152037-f427b3e689c0 h1:byYvvbfSo3+9efR4IeReh77gVs4PnNDR3AMOE9NJ7a0= github.com/ajwerner/btree v0.0.0-20211221152037-f427b3e689c0/go.mod h1:q37NoqncT41qKc048STsifIt69LfUJ8SrWWcz/yam5k= github.com/alecthomas/assert/v2 v2.0.0-alpha3 h1:pcHeMvQ3OMstAWgaeaXIAL8uzB9xMm2zlxt+/4ml8lk= +github.com/alecthomas/assert/v2 v2.0.0-alpha3/go.mod h1:+zD0lmDXTeQj7TgDgCt0ePWxb0hMC1G+PGTsTCv1B9o= github.com/alecthomas/atomic v0.1.0-alpha2 h1:dqwXmax66gXvHhsOS4pGPZKqYOlTkapELkLb3MNdlH8= github.com/alecthomas/atomic v0.1.0-alpha2/go.mod h1:zD6QGEyw49HIq19caJDc2NMXAy8rNi9ROrxtMXATfyI= github.com/alecthomas/repr v0.0.0-20210801044451-80ca428c5142 h1:8Uy0oSf5co/NZXje7U1z8Mpep++QJOldL2hs/sBQf48= +github.com/alecthomas/repr v0.0.0-20210801044451-80ca428c5142/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygvNfaDQL8= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -73,8 +77,8 @@ github.com/anacrolix/envpprof v1.3.0 h1:WJt9bpuT7A/CDCxPOv/eeZqHWlle/Y0keJUvc6tc github.com/anacrolix/envpprof v1.3.0/go.mod h1:7QIG4CaX1uexQ3tqd5+BRa/9e2D02Wcertl6Yh0jCB0= github.com/anacrolix/fuse v0.2.0 h1:pc+To78kI2d/WUjIyrsdqeJQAesuwpGxlI3h1nAv3Do= github.com/anacrolix/fuse v0.2.0/go.mod h1:Kfu02xBwnySDpH3N23BmrP3MDfwAQGRLUCj6XyeOvBQ= -github.com/anacrolix/generics v0.0.0-20230816105729-c755655aee45 h1:Kmcl3I9K2+5AdnnR7hvrnVT0TLeFWWMa9bxnm55aVIg= -github.com/anacrolix/generics v0.0.0-20230816105729-c755655aee45/go.mod h1:ff2rHB/joTV03aMSSn/AZNnaIpUw0h3njetGsaXcMy8= +github.com/anacrolix/generics v0.0.0-20230911070922-5dd7545c6b13 h1:qwOprPTDMM3BASJRf84mmZnTXRsPGGJ8xoHKQS7m3so= +github.com/anacrolix/generics v0.0.0-20230911070922-5dd7545c6b13/go.mod h1:ff2rHB/joTV03aMSSn/AZNnaIpUw0h3njetGsaXcMy8= github.com/anacrolix/go-libutp v1.3.1 h1:idJzreNLl+hNjGC3ZnUOjujEaryeOGgkwHLqSGoige0= github.com/anacrolix/go-libutp v1.3.1/go.mod h1:heF41EC8kN0qCLMokLBVkB8NXiLwx3t8R8810MTNI5o= github.com/anacrolix/log v0.3.0/go.mod h1:lWvLTqzAnCWPJA08T2HCstZi0L1y2Wyvm3FJgwU9jwU= @@ -96,13 +100,15 @@ github.com/anacrolix/missinggo/perf v1.0.0/go.mod h1:ljAFWkBuzkO12MQclXzZrosP5ur github.com/anacrolix/missinggo/v2 v2.2.0/go.mod h1:o0jgJoYOyaoYQ4E2ZMISVa9c88BbUBVQQW4QeRkNCGY= github.com/anacrolix/missinggo/v2 v2.5.1/go.mod h1:WEjqh2rmKECd0t1VhQkLGTdIWXO6f6NLjp5GlMZ+6FA= github.com/anacrolix/missinggo/v2 v2.5.2/go.mod h1:yNvsLrtZYRYCOI+KRH/JM8TodHjtIE/bjOGhQaLOWIE= -github.com/anacrolix/missinggo/v2 v2.7.2-0.20230527121029-a582b4f397b9 h1:W/oGeHhYwxueeiDjQfmK9G+X9M2xJgfTtow62v0TWAs= -github.com/anacrolix/missinggo/v2 v2.7.2-0.20230527121029-a582b4f397b9/go.mod h1:mIEtp9pgaXqt8VQ3NQxFOod/eQ1H0D1XsZzKUQfwtac= +github.com/anacrolix/missinggo/v2 v2.7.3 h1:Ee//CmZBMadeNiYB/hHo9ly2PFOEZ4Fhsbnug3rDAIE= +github.com/anacrolix/missinggo/v2 v2.7.3/go.mod h1:mIEtp9pgaXqt8VQ3NQxFOod/eQ1H0D1XsZzKUQfwtac= github.com/anacrolix/mmsg v0.0.0-20180515031531-a4a3ba1fc8bb/go.mod h1:x2/ErsYUmT77kezS63+wzZp8E3byYB0gzirM/WMBLfw= github.com/anacrolix/mmsg v1.0.0 h1:btC7YLjOn29aTUAExJiVUhQOuf/8rhm+/nWCMAnL3Hg= github.com/anacrolix/mmsg v1.0.0/go.mod h1:x8kRaJY/dCrY9Al0PEcj1mb/uFHwP6GCJ9fLl4thEPc= github.com/anacrolix/multiless v0.3.0 h1:5Bu0DZncjE4e06b9r1Ap2tUY4Au0NToBP5RpuEngSis= github.com/anacrolix/multiless v0.3.0/go.mod h1:TrCLEZfIDbMVfLoQt5tOoiBS/uq4y8+ojuEVVvTNPX4= +github.com/anacrolix/possum/go v0.0.0-20240222034319-2fe0737d4315 h1:bwdpi8LWvEHrNgsu44SKmCzLeZ5iEH4rwmNafDdRHJ4= +github.com/anacrolix/possum/go v0.0.0-20240222034319-2fe0737d4315/go.mod h1:pw5HEMBSiL+otYzHe4q5jGaVuy5unl+Mt4Bx6SDemW8= github.com/anacrolix/squirrel v0.6.0 h1:ovfWW42wcGzrVYYI9s56pEYzfeTwtXxCCvSd+KwvUEA= github.com/anacrolix/squirrel v0.6.0/go.mod h1:60vdNPUbK1jYWePp39Wqn9whHm12Yb9JEuwOXzLMDuY= github.com/anacrolix/stm v0.2.0/go.mod h1:zoVQRvSiGjGoTmbM0vSLIiaKjWtNPeTvXUSdJQA4hsg= @@ -301,6 +307,7 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4Zs github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= +github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo= github.com/huandu/xstrings v1.2.0/go.mod h1:DvyZB1rfVYsBIigL8HwpZgxHwXozlTgGqn63UyNX5k4= @@ -469,6 +476,7 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1 github.com/smartystreets/assertions v0.0.0-20190215210624-980c5ac6f3ac/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= github.com/smartystreets/goconvey v0.0.0-20190306220146-200a235640ff/go.mod h1:KSQcGKpxUMHk3nbYzs/tIBAM2iDooCn0BmttHOJEbLs= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stephens2424/writerset v1.0.2/go.mod h1:aS2JhsMn6eA7e82oNmW4rfsgAOp9COBTTl8mzkwADnc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -525,6 +533,7 @@ go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqe go.opentelemetry.io/proto/otlp v0.18.0 h1:W5hyXNComRa23tGpKwG+FRAc4rfF6ZUg1JReK+QHS80= go.opentelemetry.io/proto/otlp v0.18.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= +go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= diff --git a/storage/piece-resource.go b/storage/piece-resource.go index 5327f31d..a891e9d2 100644 --- a/storage/piece-resource.go +++ b/storage/piece-resource.go @@ -3,11 +3,11 @@ package storage import ( "bytes" "fmt" + "github.com/anacrolix/sync" "io" "path" "sort" "strconv" - "sync" "github.com/anacrolix/missinggo/v2/resource" diff --git a/storage/possum/possum-provider.go b/storage/possum/possum-provider.go new file mode 100644 index 00000000..e67cdada --- /dev/null +++ b/storage/possum/possum-provider.go @@ -0,0 +1,135 @@ +//go:build !android + +package possumTorrentStorage + +import ( + "cmp" + "fmt" + "github.com/anacrolix/log" + possum "github.com/anacrolix/possum/go" + possumResource "github.com/anacrolix/possum/go/resource" + "github.com/anacrolix/torrent/storage" + "io" + "sort" + "strconv" +) + +// Extends possum resource.Provider with an efficient implementation of torrent +// storage.ConsecutiveChunkReader. +type Provider struct { + possumResource.Provider + Logger log.Logger +} + +var _ storage.ConsecutiveChunkReader = Provider{} + +// Sorts by a precomputed key but swaps on another slice at the same time. +type keySorter[T any, K cmp.Ordered] struct { + orig []T + keys []K +} + +func (o keySorter[T, K]) Len() int { + return len(o.keys) +} + +func (o keySorter[T, K]) Less(i, j int) bool { + return o.keys[i] < o.keys[j] +} + +func (o keySorter[T, K]) Swap(i, j int) { + o.keys[i], o.keys[j] = o.keys[j], o.keys[i] + o.orig[i], o.orig[j] = o.orig[j], o.orig[i] +} + +// TODO: Should the parent ReadConsecutiveChunks method take the expected number of bytes to avoid +// trying to read discontinuous or incomplete sequences of chunks? +func (p Provider) ReadConsecutiveChunks(prefix string) (rc io.ReadCloser, err error) { + p.Logger.Levelf(log.Debug, "ReadConsecutiveChunks(%q)", prefix) + //debug.PrintStack() + pr, err := p.Handle.NewReader() + if err != nil { + return + } + defer func() { + if err != nil { + pr.End() + } + }() + items, err := pr.ListItems(prefix) + if err != nil { + return + } + keys := make([]int64, 0, len(items)) + for _, item := range items { + var i int64 + offsetStr := item.Key + i, err = strconv.ParseInt(offsetStr, 10, 64) + if err != nil { + err = fmt.Errorf("failed to parse offset %q: %w", offsetStr, err) + return + } + keys = append(keys, i) + } + sort.Sort(keySorter[possum.Item, int64]{items, keys}) + offset := int64(0) + consValues := make([]consecutiveValue, 0, len(items)) + for i, item := range items { + itemOffset := keys[i] + if itemOffset > offset { + // We can't provide a continuous read. + break + } + if itemOffset+item.Stat.Size() <= offset { + // This item isn't needed + continue + } + var v possum.Value + v, err = pr.Add(prefix + item.Key) + if err != nil { + return + } + consValues = append(consValues, consecutiveValue{ + pv: v, + offset: itemOffset, + size: item.Stat.Size(), + }) + offset += item.Stat.Size() - (offset - itemOffset) + } + err = pr.Begin() + if err != nil { + return + } + rc, pw := io.Pipe() + go func() { + defer pr.End() + err := p.writeConsecutiveValues(consValues, pw) + err = pw.CloseWithError(err) + if err != nil { + panic(err) + } + }() + return +} + +type consecutiveValue struct { + pv possum.Value + offset int64 + size int64 +} + +func (pp Provider) writeConsecutiveValues( + values []consecutiveValue, pw *io.PipeWriter, +) (err error) { + off := int64(0) + for _, v := range values { + var n int64 + valueOff := off - v.offset + n, err = io.Copy(pw, io.NewSectionReader(v.pv, valueOff, v.size-valueOff)) + if err != nil { + return + } + off += n + } + return nil +} diff --git a/storage/possum/possum_test.go b/storage/possum/possum_test.go new file mode 100644 index 00000000..6bd350b1 --- /dev/null +++ b/storage/possum/possum_test.go @@ -0,0 +1,30 @@ +package possumTorrentStorage + +import ( + g "github.com/anacrolix/generics" + "github.com/anacrolix/log" + possum "github.com/anacrolix/possum/go" + possumResource "github.com/anacrolix/possum/go/resource" + "github.com/anacrolix/torrent/storage" + test_storage "github.com/anacrolix/torrent/storage/test" + "testing" +) + +// This should be made to mirror the benchmarks for sqlite storage. +func BenchmarkProvider(b *testing.B) { + possumDir, err := possum.Open(b.TempDir()) + if err != nil { + b.Fatal(err) + } + possumDir.SetInstanceLimits(possum.Limits{ + DisableHolePunching: false, + MaxValueLengthSum: g.Some[uint64](test_storage.DefaultPieceSize * test_storage.DefaultNumPieces / 2), + }) + defer possumDir.Close() + possumProvider := possumResource.Provider{Handle: possumDir} + possumTorrentProvider := Provider{Provider: possumProvider, Logger: log.Default} + clientStorageImpl := storage.NewResourcePiecesOpts( + possumTorrentProvider, + storage.ResourcePiecesOpts{LeaveIncompleteChunks: true}) + test_storage.BenchmarkPieceMarkComplete(b, clientStorageImpl, test_storage.DefaultPieceSize, test_storage.DefaultNumPieces, 0) +}