From a63d3a16a6518bc7c074d54c1a6bf8d68ac0dbbc Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Wed, 2 Jul 2025 15:12:17 +1000 Subject: [PATCH] Respect rate limiting active webseed requests --- webseed-peer.go | 5 ++--- webseed-requesting.go | 11 ++++++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/webseed-peer.go b/webseed-peer.go index a8f453c9..b453deec 100644 --- a/webseed-peer.go +++ b/webseed-peer.go @@ -147,8 +147,7 @@ func (ws *webseedPeer) runRequest(webseedRequest *webseedRequest) { torrent.Add("webseed request error count", 1) // This used to occur only on webseed.ErrTooFast but I think it makes sense to slow down any // kind of error. Pausing here will starve the available requester slots which slows things - // down. TODO: I don't think this will help anymore. Need to register a reduced concurrency - // available for a host/cost key. + // down. select { case <-ws.peer.closed.Done(): case <-time.After(time.Duration(rand.Int63n(int64(10 * time.Second)))): @@ -161,7 +160,7 @@ func (ws *webseedPeer) runRequest(webseedRequest *webseedRequest) { if err != nil { ws.peer.onNeedUpdateRequests("webseedPeer request errored") } - ws.peer.t.cl.updateWebSeedRequests("webseedPeer request completed") + ws.peer.t.cl.updateWebseedRequestsWithReason("webseedPeer request completed") locker.Unlock() } diff --git a/webseed-requesting.go b/webseed-requesting.go index 9bb9bbbb..0a7c0333 100644 --- a/webseed-requesting.go +++ b/webseed-requesting.go @@ -122,8 +122,13 @@ func (cl *Client) globalUpdateWebSeedRequests() { value.existingWebseedRequest.Cancel() } - for _, requestKeys := range plan.byCost { + for costKey, requestKeys := range plan.byCost { for _, requestKey := range requestKeys { + // This could happen if a request is cancelled but hasn't removed itself from the active + // list yet. This helps with backpressure as the requests can sleep to rate limit. + if !cl.underWebSeedHttpRequestLimit(costKey) { + break + } if g.MapContains(existingRequests, requestKey) { continue } @@ -204,7 +209,8 @@ func (cl *Client) iterWebseed() iter.Seq2[webseedUniqueRequestKey, webseedReques } -func (cl *Client) updateWebSeedRequests(reason updateRequestReason) { +func (cl *Client) updateWebseedRequestsWithReason(reason updateRequestReason) { + // Should we wrap this with pprof labels? cl.updateWebseedRequests() } @@ -244,7 +250,6 @@ func (cl *Client) iterCurrentWebseedRequests() iter.Seq2[webseedUniqueRequestKey func (cl *Client) updateWebseedRequests() { cl.globalUpdateWebSeedRequests() - // Should have already run to get here. cl.webseedRequestTimer.Reset(webseedRequestUpdateTimerInterval) } -- 2.51.0