Make WebSeed max requests configurable

diff --git a/torrent.go b/torrent.go
index 312e4509f5d0f4c43f3bbe38b693ff7772467d15..f6d39523244d4c8f7224dbeff55bc838e238af4c 100644 (file)
--- a/torrent.go
+++ b/torrent.go
@@ -2888,6 +2888,13 @@ func (t *Torrent) callbacks() *Callbacks {
 
 type AddWebSeedsOpt func(*webseed.Client)
 
+// Max concurrent requests to a WebSeed for a given torrent.
+func WebSeedTorrentMaxRequests(maxRequests int) AddWebSeedsOpt {
+       return func(c *webseed.Client) {
+               c.MaxRequests = maxRequests
+       }
+}
+
 // Sets the WebSeed trailing path escaper for a webseed.Client.
 func WebSeedPathEscaper(custom webseed.PathEscaper) AddWebSeedsOpt {
        return func(c *webseed.Client) {
@@ -2903,37 +2910,34 @@ func (t *Torrent) AddWebSeeds(urls []string, opts ...AddWebSeedsOpt) {
        }
 }
 
-func (t *Torrent) addWebSeed(url string, opts ...AddWebSeedsOpt) {
+// Returns true if the WebSeed was newly added with the provided configuration.
+func (t *Torrent) addWebSeed(url string, opts ...AddWebSeedsOpt) bool {
        if t.cl.config.DisableWebseeds {
-               return
+               return false
        }
        if _, ok := t.webSeeds[url]; ok {
-               return
+               return false
        }
        // I don't think Go http supports pipelining requests. However, we can have more ready to go
        // right away. This value should be some multiple of the number of connections to a host. I
        // would expect that double maxRequests plus a bit would be appropriate. This value is based on
        // downloading Sintel (08ada5a7a6183aae1e09d831df6748d566095a10) from
        // "https://webtorrent.io/torrents/".
-       const maxRequests = 16
+       const defaultMaxRequests = 16
        ws := webseedPeer{
                peer: Peer{
                        t:                        t,
                        outgoing:                 true,
                        Network:                  "http",
                        reconciledHandshakeStats: true,
-                       // This should affect how often we have to recompute requests for this peer. Note that
-                       // because we can request more than 1 thing at a time over HTTP, we will hit the low
-                       // requests mark more often, so recomputation is probably sooner than with regular peer
-                       // conns. ~4x maxRequests would be about right.
-                       PeerMaxRequests: 128,
                        // TODO: Set ban prefix?
                        RemoteAddr: remoteAddrFromUrl(url),
                        callbacks:  t.callbacks(),
                },
                client: webseed.Client{
-                       HttpClient: t.cl.httpClient,
-                       Url:        url,
+                       HttpClient:  t.cl.httpClient,
+                       Url:         url,
+                       MaxRequests: defaultMaxRequests,
                        ResponseBodyWrapper: func(r io.Reader) io.Reader {
                                return &rateLimitedReader{
                                        l: t.cl.config.DownloadRateLimiter,
@@ -2941,15 +2945,20 @@ func (t *Torrent) addWebSeed(url string, opts ...AddWebSeedsOpt) {
                                }
                        },
                },
-               activeRequests: make(map[Request]webseed.Request, maxRequests),
        }
        ws.peer.initRequestState()
        for _, opt := range opts {
                opt(&ws.client)
        }
+       g.MakeMapWithCap(&ws.activeRequests, ws.client.MaxRequests)
+       // This should affect how often we have to recompute requests for this peer. Note that
+       // because we can request more than 1 thing at a time over HTTP, we will hit the low
+       // requests mark more often, so recomputation is probably sooner than with regular peer
+       // conns. ~4x maxRequests would be about right.
+       ws.peer.PeerMaxRequests = 4 * ws.client.MaxRequests
        ws.peer.initUpdateRequestsTimer()
        ws.requesterCond.L = t.cl.locker()
-       for i := 0; i < maxRequests; i += 1 {
+       for i := 0; i < ws.client.MaxRequests; i += 1 {
                go ws.requester(i)
        }
        for _, f := range t.callbacks().NewPeer {
@@ -2964,6 +2973,7 @@ func (t *Torrent) addWebSeed(url string, opts ...AddWebSeedsOpt) {
        }
        t.webSeeds[url] = &ws.peer
        ws.peer.updateRequests("Torrent.addWebSeed")
+       return true
 }
 
 func (t *Torrent) peerIsActive(p *Peer) (active bool) {

diff --git a/webseed/client.go b/webseed/client.go
index e6dce7a80cb7b4aa975121f0fcb3326b5ac7b287..288c0fb782170f32243c1b2adda53d5487d6112d 100644 (file)
--- a/webseed/client.go
+++ b/webseed/client.go
@@ -39,8 +39,11 @@ func (r Request) Cancel() {
 type Client struct {
        HttpClient *http.Client
        Url        string
-       fileIndex  segments.Index
-       info       *metainfo.Info
+       // Max concurrent requests to a WebSeed for a given torrent.
+       MaxRequests int
+
+       fileIndex segments.Index
+       info      *metainfo.Info
        // The pieces we can request with the Url. We're more likely to ban/block at the file-level
        // given that's how requests are mapped to webseeds, but the torrent.Client works at the piece
        // level. We can map our file-level adjustments to the pieces here. This probably need to be