From: Matt Joiner Date: Tue, 13 May 2025 01:39:12 +0000 (+1000) Subject: Restructure things ready for new webseed algorithm X-Git-Tag: v1.59.0~145 X-Git-Url: http://www.git.stargrave.org/?a=commitdiff_plain;h=0665f26a72a1c7ee59738f5da96d92f140a4e700;p=btrtrc.git Restructure things ready for new webseed algorithm --- diff --git a/client-piece-request-order.go b/client-piece-request-order.go index 787fc7ef..b4ad07da 100644 --- a/client-piece-request-order.go +++ b/client-piece-request-order.go @@ -1,19 +1,51 @@ package torrent import ( + requestStrategy "github.com/anacrolix/torrent/request-strategy" "github.com/anacrolix/torrent/storage" ) +// clientPieceRequestOrderKey is a key for the piece request order map in the Client. type clientPieceRequestOrderKeyTypes interface { storage.TorrentCapacity | *Torrent } +// clientPieceRequestOrderKey is a generic key type for the piece request order map. type clientPieceRequestOrderKey[T clientPieceRequestOrderKeyTypes] struct { inner T } -func (me clientPieceRequestOrderKey[T]) isAClientPieceRequestOrderKeyType() {} +// clientPieceRequestOrderRegularTorrentKey is a concrete key type for regular torrents. +type clientPieceRequestOrderRegularTorrentKey clientPieceRequestOrderKey[*Torrent] + +func (c clientPieceRequestOrderRegularTorrentKey) getRequestStrategyInput(cl *Client) requestStrategy.Input { + return requestStrategyInputSingleTorrent{ + requestStrategyInputCommon: cl.getRequestStrategyInputCommon(), + t: c.inner, + } +} + +// clientPieceRequestOrderSharedStorageTorrentKey is a concrete key type for shared storage torrents. +type clientPieceRequestOrderSharedStorageTorrentKey clientPieceRequestOrderKey[storage.TorrentCapacity] + +func (c clientPieceRequestOrderSharedStorageTorrentKey) getRequestStrategyInput(cl *Client) requestStrategy.Input { + return requestStrategyInputMultiTorrent{ + requestStrategyInputCommon: cl.getRequestStrategyInputCommon(), + torrents: cl.torrentsByShortHash, + capFunc: c.inner, + } +} type clientPieceRequestOrderKeySumType interface { - isAClientPieceRequestOrderKeyType() + // getRequestStrategyInput returns the request strategy input for the torrent. It depends on the + // storage capacity arrangements which is the defining differentiator for the client piece + // request order keys. This code reads like that stupid second-year software design course I + // failed 3 times. + getRequestStrategyInput(cl *Client) requestStrategy.Input +} + +type clientPieceRequestOrderValue struct { + // TODO: Check if we actually ended up needing this? + torrents map[*Torrent]struct{} + pieces *requestStrategy.PieceRequestOrder } diff --git a/client.go b/client.go index 5627a8dc..4fffd5fd 100644 --- a/client.go +++ b/client.go @@ -43,7 +43,6 @@ import ( "github.com/anacrolix/torrent/metainfo" "github.com/anacrolix/torrent/mse" pp "github.com/anacrolix/torrent/peer_protocol" - request_strategy "github.com/anacrolix/torrent/request-strategy" "github.com/anacrolix/torrent/storage" "github.com/anacrolix/torrent/tracker" "github.com/anacrolix/torrent/types/infohash" @@ -86,7 +85,9 @@ type Client struct { // info has been obtained, there's no knowing if an infohash belongs to v1 or v2. torrentsByShortHash map[InfoHash]*Torrent - pieceRequestOrder map[clientPieceRequestOrderKeySumType]*request_strategy.PieceRequestOrder + // Piece request orderings grouped by storage. Value is value type because all fields are + // references. + pieceRequestOrder map[clientPieceRequestOrderKeySumType]clientPieceRequestOrderValue acceptLimiter map[ipStr]int numHalfOpen int diff --git a/piece.go b/piece.go index 074f6601..17a4ed89 100644 --- a/piece.go +++ b/piece.go @@ -296,6 +296,7 @@ func (p *Piece) requestIndexOffset() RequestIndex { return p.t.pieceRequestIndexOffset(p.index) } +// TODO: Make this peer-only? func (p *Piece) availability() int { return len(p.t.connsWithAllPieces) + p.relativeAvailability } diff --git a/request-strategy-impls.go b/request-strategy-impls.go index dfcec28a..fd52db64 100644 --- a/request-strategy-impls.go +++ b/request-strategy-impls.go @@ -30,6 +30,9 @@ func (r requestStrategyInputMultiTorrent) Capacity() (int64, bool) { return (*r.capFunc)() } +// I don't think we need this for correctness purposes, but it must be faster to look up the Torrent +// input because it's locked to a given Torrent. It would be easy enough to drop in the +// multi-torrent version in this place and compare. type requestStrategyInputSingleTorrent struct { requestStrategyInputCommon t *Torrent @@ -45,31 +48,16 @@ func (r requestStrategyInputSingleTorrent) Capacity() (cap int64, capped bool) { var _ request_strategy.Input = requestStrategyInputSingleTorrent{} +// getRequestStrategyInputCommon returns request strategy Input implementation common to all inputs. func (cl *Client) getRequestStrategyInputCommon() requestStrategyInputCommon { return requestStrategyInputCommon{cl.config.MaxUnverifiedBytes} } -// Returns what is necessary to run request_strategy.GetRequestablePieces for primaryTorrent. -func (cl *Client) getRequestStrategyInput(primaryTorrent *Torrent) (input request_strategy.Input) { - if !primaryTorrent.hasStorageCap() { - return requestStrategyInputSingleTorrent{ - requestStrategyInputCommon: cl.getRequestStrategyInputCommon(), - t: primaryTorrent, - } - } else { - return requestStrategyInputMultiTorrent{ - requestStrategyInputCommon: cl.getRequestStrategyInputCommon(), - // TODO: Check this is an appropriate key - torrents: cl.torrentsByShortHash, - capFunc: primaryTorrent.storage.Capacity, - } - } -} - func (t *Torrent) getRequestStrategyInput() request_strategy.Input { - return t.cl.getRequestStrategyInput(t) + return t.clientPieceRequestOrderKey().getRequestStrategyInput(t.cl) } +// Wraps a Torrent to provide request-strategy.Torrent interface. type requestStrategyTorrent struct { t *Torrent } diff --git a/request-strategy/order.go b/request-strategy/order.go index b0686aff..bffe6a7a 100644 --- a/request-strategy/order.go +++ b/request-strategy/order.go @@ -19,6 +19,9 @@ type ( ChunkSpec = types.ChunkSpec ) +// Piece request ordering factoring in storage limits, user-assigned priority, network availability. +// Is it missing the random piece affinity assigned per torrent? Can we do that deterministically +// per-client? func pieceOrderLess(i, j *PieceRequestOrderItem) multiless.Computation { return multiless.New().Int( int(j.State.Priority), int(i.State.Priority), @@ -39,12 +42,18 @@ func pieceOrderLess(i, j *PieceRequestOrderItem) multiless.Computation { }) } +// Returns true if the piece should be considered against the unverified bytes limit. This is +// based on whether the callee intends to request from the piece. Pieces submitted to this +// callback passed Piece.Request and so are ready for immediate download. +type RequestPieceFunc func(ih metainfo.Hash, pieceIndex int, orderState PieceRequestOrderState) bool + // Calls f with requestable pieces in order. func GetRequestablePieces( input Input, pro *PieceRequestOrder, // Returns true if the piece should be considered against the unverified bytes limit. This is - // based on whether the callee intends to request from the piece. - requestPiece func(ih metainfo.Hash, pieceIndex int, orderState PieceRequestOrderState) bool, + // based on whether the callee intends to request from the piece. Pieces submitted to this + // callback passed Piece.Request and so are ready for immediate download. + requestPiece RequestPieceFunc, ) { // Storage capacity left for this run, keyed by the storage capacity pointer on the storage // TorrentImpl. A nil value means no capacity limit. diff --git a/request-strategy/piece.go b/request-strategy/piece.go index 02c9ff65..4aff5272 100644 --- a/request-strategy/piece.go +++ b/request-strategy/piece.go @@ -5,6 +5,9 @@ type Piece interface { // is currently being hashed, or already complete. Request() bool // Whether the piece should be counted towards the unverified bytes limit. The intention is to - // prevent pieces being starved from the opportunity to move to the completed state. + // prevent pieces being starved from the opportunity to move to the completed state. Pieces that + // are in an overhead state like being hashed, queued, or having metadata modified are here. If + // we didn't count them we could race ahead downloading and leave lots of pieces stuck in an + // intermediate state. CountUnverified() bool } diff --git a/requesting.go b/requesting.go index bedb07be..0d72733d 100644 --- a/requesting.go +++ b/requesting.go @@ -4,6 +4,7 @@ import ( "context" "encoding/gob" "fmt" + "github.com/anacrolix/torrent/metainfo" "reflect" "runtime/pprof" "time" @@ -168,6 +169,16 @@ type desiredRequestState struct { Interested bool } +func (cl *Client) getRequestablePieces(key clientPieceRequestOrderKeySumType, f requestStrategy.RequestPieceFunc) { + input := key.getRequestStrategyInput(cl) + order := cl.pieceRequestOrder[key].pieces + requestStrategy.GetRequestablePieces(input, order, f) +} + +func (t *Torrent) getRequestablePieces(f requestStrategy.RequestPieceFunc) { + t.cl.getRequestablePieces(t.clientPieceRequestOrderKey(), f) +} + // This gets the best-case request state. That means handling pieces limited by capacity, preferring // earlier pieces, low availability etc. It pays no attention to existing requests on the peer or // other peers. Those are handled later. @@ -182,7 +193,6 @@ func (p *PeerConn) getDesiredRequestState() (desired desiredRequestState) { if t.dataDownloadDisallowed.Bool() { return } - input := t.getRequestStrategyInput() requestHeap := desiredPeerRequests{ peer: &p.Peer, pieceStates: t.requestPieceStates, @@ -192,10 +202,8 @@ func (p *PeerConn) getDesiredRequestState() (desired desiredRequestState) { t.logPieceRequestOrder() // Caller-provided allocation for roaring bitmap iteration. var it typedRoaring.Iterator[RequestIndex] - requestStrategy.GetRequestablePieces( - input, - t.getPieceRequestOrder(), - func(ih InfoHash, pieceIndex int, pieceExtra requestStrategy.PieceRequestOrderState) bool { + t.getRequestablePieces( + func(ih metainfo.Hash, pieceIndex int, pieceExtra requestStrategy.PieceRequestOrderState) bool { if ih != *t.canonicalShortInfohash() { return false } diff --git a/torrent-piece-request-order.go b/torrent-piece-request-order.go index 7a9cf221..f23144aa 100644 --- a/torrent-piece-request-order.go +++ b/torrent-piece-request-order.go @@ -4,7 +4,6 @@ import ( g "github.com/anacrolix/generics" request_strategy "github.com/anacrolix/torrent/request-strategy" - "github.com/anacrolix/torrent/storage" ) // It's probably possible to track whether the piece moves around in the btree to be more efficient @@ -19,23 +18,23 @@ func (t *Torrent) updatePieceRequestOrderPiece(pieceIndex int) (changed bool) { } key := t.pieceRequestOrderKey(pieceIndex) if t.hasStorageCap() { - return pro.Update(key, t.requestStrategyPieceOrderState(pieceIndex)) + return pro.pieces.Update(key, t.requestStrategyPieceOrderState(pieceIndex)) } pending := !t.ignorePieceForRequests(pieceIndex) if pending { newState := t.requestStrategyPieceOrderState(pieceIndex) - old := pro.Add(key, newState) + old := pro.pieces.Add(key, newState) return old.Ok && old.Value != newState } else { - return pro.Delete(key) + return pro.pieces.Delete(key) } } func (t *Torrent) clientPieceRequestOrderKey() clientPieceRequestOrderKeySumType { if t.storage.Capacity == nil { - return clientPieceRequestOrderKey[*Torrent]{t} + return clientPieceRequestOrderRegularTorrentKey{t} } - return clientPieceRequestOrderKey[storage.TorrentCapacity]{t.storage.Capacity} + return clientPieceRequestOrderSharedStorageTorrentKey{t.storage.Capacity} } func (t *Torrent) deletePieceRequestOrder() { @@ -45,10 +44,11 @@ func (t *Torrent) deletePieceRequestOrder() { cpro := t.cl.pieceRequestOrder key := t.clientPieceRequestOrderKey() pro := cpro[key] - for i := 0; i < t.numPieces(); i++ { - pro.Delete(t.pieceRequestOrderKey(i)) + for i := range t.numPieces() { + pro.pieces.Delete(t.pieceRequestOrderKey(i)) } - if pro.Len() == 0 { + delete(pro.torrents, t) + if pro.pieces.Len() == 0 { delete(cpro, key) } } @@ -60,9 +60,14 @@ func (t *Torrent) initPieceRequestOrder() { g.MakeMapIfNil(&t.cl.pieceRequestOrder) key := t.clientPieceRequestOrderKey() cpro := t.cl.pieceRequestOrder - if cpro[key] == nil { - cpro[key] = request_strategy.NewPieceOrder(request_strategy.NewAjwernerBtree(), t.numPieces()) + if _, ok := cpro[key]; !ok { + value := clientPieceRequestOrderValue{ + pieces: request_strategy.NewPieceOrder(request_strategy.NewAjwernerBtree(), t.numPieces()), + } + g.MakeMap(&value.torrents) + cpro[key] = value } + g.MapMustAssignNew(cpro[key].torrents, t, struct{}{}) } func (t *Torrent) addRequestOrderPiece(i int) { @@ -80,5 +85,5 @@ func (t *Torrent) getPieceRequestOrder() *request_strategy.PieceRequestOrder { if t.storage == nil { return nil } - return t.cl.pieceRequestOrder[t.clientPieceRequestOrderKey()] + return t.cl.pieceRequestOrder[t.clientPieceRequestOrderKey()].pieces } diff --git a/webseed-requesting.go b/webseed-requesting.go new file mode 100644 index 00000000..bbcbc9b9 --- /dev/null +++ b/webseed-requesting.go @@ -0,0 +1,25 @@ +package torrent + +import ( + "github.com/anacrolix/torrent/metainfo" + requestStrategy "github.com/anacrolix/torrent/request-strategy" +) + +/* +- Go through all the requestable pieces in order of priority, availability, whether there are peer requests, partial, infohash. +- For each piece calculate files involved. Record each file not seen before and the piece index. +- Cancel any outstanding requests that don't match a final file/piece-index pair. +- Initiate missing requests that fit into the available limits. +*/ +func (cl *Client) updateWebSeedRequests() { + for key, value := range cl.pieceRequestOrder { + input := key.getRequestStrategyInput(cl) + requestStrategy.GetRequestablePieces( + input, + value.pieces, + func(ih metainfo.Hash, pieceIndex int, orderState requestStrategy.PieceRequestOrderState) bool { + return true + }, + ) + } +}