Respect rate limiting active webseed requests

diff --git a/webseed-peer.go b/webseed-peer.go
index a8f453c919dbde02da48e3369681917ce6c0a7ea..b453deec60eeec1b0b0d6e6e1e91d55b9e1e0f4c 100644 (file)
--- a/webseed-peer.go
+++ b/webseed-peer.go
@@ -147,8 +147,7 @@ func (ws *webseedPeer) runRequest(webseedRequest *webseedRequest) {
                torrent.Add("webseed request error count", 1)
                // This used to occur only on webseed.ErrTooFast but I think it makes sense to slow down any
                // kind of error. Pausing here will starve the available requester slots which slows things
-               // down. TODO: I don't think this will help anymore. Need to register a reduced concurrency
-               // available for a host/cost key.
+               // down.
                select {
                case <-ws.peer.closed.Done():
                case <-time.After(time.Duration(rand.Int63n(int64(10 * time.Second)))):
@@ -161,7 +160,7 @@ func (ws *webseedPeer) runRequest(webseedRequest *webseedRequest) {
        if err != nil {
                ws.peer.onNeedUpdateRequests("webseedPeer request errored")
        }
-       ws.peer.t.cl.updateWebSeedRequests("webseedPeer request completed")
+       ws.peer.t.cl.updateWebseedRequestsWithReason("webseedPeer request completed")
        locker.Unlock()
 }
 

diff --git a/webseed-requesting.go b/webseed-requesting.go
index 9bb9bbbb8c5c1d6d90a292f4b64116c25d4075b3..0a7c0333e86478bda77b6cc4cd5c6a1ed4d788f4 100644 (file)
--- a/webseed-requesting.go
+++ b/webseed-requesting.go
@@ -122,8 +122,13 @@ func (cl *Client) globalUpdateWebSeedRequests() {
                value.existingWebseedRequest.Cancel()
        }
 
-       for _, requestKeys := range plan.byCost {
+       for costKey, requestKeys := range plan.byCost {
                for _, requestKey := range requestKeys {
+                       // This could happen if a request is cancelled but hasn't removed itself from the active
+                       // list yet. This helps with backpressure as the requests can sleep to rate limit.
+                       if !cl.underWebSeedHttpRequestLimit(costKey) {
+                               break
+                       }
                        if g.MapContains(existingRequests, requestKey) {
                                continue
                        }
@@ -204,7 +209,8 @@ func (cl *Client) iterWebseed() iter.Seq2[webseedUniqueRequestKey, webseedReques
 
 }
 
-func (cl *Client) updateWebSeedRequests(reason updateRequestReason) {
+func (cl *Client) updateWebseedRequestsWithReason(reason updateRequestReason) {
+       // Should we wrap this with pprof labels?
        cl.updateWebseedRequests()
 }
 
@@ -244,7 +250,6 @@ func (cl *Client) iterCurrentWebseedRequests() iter.Seq2[webseedUniqueRequestKey
 
 func (cl *Client) updateWebseedRequests() {
        cl.globalUpdateWebSeedRequests()
-       // Should have already run to get here.
        cl.webseedRequestTimer.Reset(webseedRequestUpdateTimerInterval)
 }