Limit webseeds by host and reduce default to 5 per Client

diff --git a/client.go b/client.go
index f7ee73e7af9a113ba17b8f91caa26fd8f919b544..dbe0b31ea6452e310ca232fadaf58ac2e59e437f 100644 (file)
--- a/client.go
+++ b/client.go
@@ -29,6 +29,7 @@ import (
        "github.com/anacrolix/log"
        "github.com/anacrolix/missinggo/v2"
        "github.com/anacrolix/missinggo/v2/bitmap"
+       "github.com/anacrolix/missinggo/v2/panicif"
        "github.com/anacrolix/missinggo/v2/pproffd"
        "github.com/anacrolix/sync"
        "github.com/cespare/xxhash"
@@ -94,7 +95,7 @@ type Client struct {
        numHalfOpen   int
 
        websocketTrackers  websocketTrackers
-       numWebSeedRequests int
+       numWebSeedRequests map[webseedHostKeyHandle]int
 
        activeAnnounceLimiter limiter.Instance
        httpClient            *http.Client
@@ -1933,8 +1934,9 @@ func (cl *Client) Stats() ClientStats {
        return cl.statsLocked()
 }
 
-func (cl *Client) underWebSeedHttpRequestLimit() bool {
-       return cl.numWebSeedRequests < 10
+func (cl *Client) underWebSeedHttpRequestLimit(key webseedHostKeyHandle) bool {
+       panicif.Zero(key)
+       return cl.numWebSeedRequests[key] < 5
 }
 
 func (cl *Client) countWebSeedHttpRequests() (num int) {

diff --git a/torrent.go b/torrent.go
index dc9571bac119130af4f3afba1b1a94d64c03b432..1df519720a38bc144e990f164c45baca8cbd4718 100644 (file)
--- a/torrent.go
+++ b/torrent.go
@@ -19,6 +19,7 @@ import (
        "strings"
        "text/tabwriter"
        "time"
+       "unique"
        "unsafe"
 
        "github.com/RoaringBitmap/roaring"
@@ -3049,6 +3050,7 @@ func (t *Torrent) addWebSeed(url string, opts ...AddWebSeedsOpt) bool {
                                }
                        },
                },
+               hostKey: t.deriveWebSeedHostKey(url),
        }
        ws.peer.initRequestState()
        for _, opt := range opts {
@@ -3076,6 +3078,15 @@ func (t *Torrent) addWebSeed(url string, opts ...AddWebSeedsOpt) bool {
        return true
 }
 
+func (t *Torrent) deriveWebSeedHostKey(urlStr string) (ret webseedHostKeyHandle) {
+       u, err := url.Parse(urlStr)
+       if err != nil {
+               t.slogger().Warn("error parsing webseed URL", "url", urlStr, "err", err)
+               return unique.Make(webseedHostKey(urlStr))
+       }
+       return unique.Make(webseedHostKey(u.Hostname()))
+}
+
 func (t *Torrent) peerIsActive(p *Peer) (active bool) {
        t.iterPeers(func(p1 *Peer) {
                if p1 == p {

diff --git a/webseed-peer.go b/webseed-peer.go
index bedbb927e12cd10b74a361dbbf86cc1d2484f8a2..f02845a44e8ae45595b322f790a6b9bfbb11913b 100644 (file)
--- a/webseed-peer.go
+++ b/webseed-peer.go
@@ -2,7 +2,6 @@ package torrent
 
 import (
        "context"
-       "errors"
        "fmt"
        "io"
        "iter"
@@ -13,7 +12,7 @@ import (
 
        "github.com/RoaringBitmap/roaring"
        g "github.com/anacrolix/generics"
-
+       "github.com/anacrolix/missinggo/v2/panicif"
        "github.com/anacrolix/torrent/metainfo"
        pp "github.com/anacrolix/torrent/peer_protocol"
        "github.com/anacrolix/torrent/webseed"
@@ -26,6 +25,7 @@ type webseedPeer struct {
        activeRequests   map[*webseedRequest]struct{}
        locker           sync.Locker
        lastUnhandledErr time.Time
+       hostKey          webseedHostKeyHandle
 }
 
 func (me *webseedPeer) nominalMaxRequests() maxRequests {
@@ -44,7 +44,11 @@ func (me *webseedPeer) numRequests() int {
 }
 
 func (me *webseedPeer) shouldUpdateRequests() bool {
-       return me.numRequests() < me.client.MaxRequests && me.peer.t.cl.underWebSeedHttpRequestLimit()
+       return me.moreRequestsAllowed()
+}
+
+func (me *webseedPeer) moreRequestsAllowed() bool {
+       return me.numRequests() < me.client.MaxRequests && me.peer.t.cl.underWebSeedHttpRequestLimit(me.hostKey)
 }
 
 func (me *webseedPeer) updateRequests() {
@@ -133,7 +137,8 @@ func (ws *webseedPeer) spawnRequest(begin, end RequestIndex) {
                end:     end,
        }
        ws.activeRequests[&wsReq] = struct{}{}
-       ws.peer.t.cl.numWebSeedRequests++
+       panicif.Zero(ws.hostKey)
+       ws.peer.t.cl.numWebSeedRequests[ws.hostKey]++
        ws.slogger().Debug(
                "starting webseed request",
                "begin", begin,
@@ -175,19 +180,13 @@ func (ws *webseedPeer) runRequest(webseedRequest *webseedRequest) {
 
 func (ws *webseedPeer) deleteActiveRequest(wr *webseedRequest) {
        g.MustDelete(ws.activeRequests, wr)
-       ws.peer.t.cl.numWebSeedRequests--
+       ws.peer.t.cl.numWebSeedRequests[ws.hostKey]--
 }
 
 func (ws *webseedPeer) spawnRequests() {
        next, stop := iter.Pull(ws.inactiveRequests())
        defer stop()
-       for {
-               if !ws.peer.t.cl.underWebSeedHttpRequestLimit() {
-                       break
-               }
-               if ws.numRequests() >= ws.client.MaxRequests {
-                       break
-               }
+       for ws.moreRequestsAllowed() {
                req, ok := next()
                if !ok {
                        break

diff --git a/webseed-requesting.go b/webseed-requesting.go
index 66893c119c182c2a5a16cc6ff45524a855dd598f..607cba409f1762a5f8ef65c9d38cd9cd96bfbfe7 100644 (file)
--- a/webseed-requesting.go
+++ b/webseed-requesting.go
@@ -1,10 +1,17 @@
 package torrent
 
 import (
+       "unique"
+
        "github.com/anacrolix/torrent/metainfo"
        requestStrategy "github.com/anacrolix/torrent/request-strategy"
 )
 
+type (
+       webseedHostKey       string
+       webseedHostKeyHandle = unique.Handle[webseedHostKey]
+)
+
 /*
 - Go through all the requestable pieces in order of priority, availability, whether there are peer requests, partial, infohash.
 - For each piece calculate files involved. Record each file not seen before and the piece index.