From 02cc723750e69cd4ce327ced62d595b6c13c1479 Mon Sep 17 00:00:00 2001 From: afjoseph <7126721+afjoseph@users.noreply.github.com> Date: Thu, 21 Apr 2022 16:21:29 +0200 Subject: [PATCH] [webseed] Add a custom URL encoder for webseeds --- client.go | 2 +- spec.go | 4 ++++ torrent.go | 20 +++++++++++++++++--- webseed/client.go | 11 ++++++++++- webseed/request.go | 36 ++++++++++++++++++++++++++++++++---- webseed/request_test.go | 8 ++++---- 6 files changed, 68 insertions(+), 13 deletions(-) diff --git a/client.go b/client.go index ee8261f3..7d763c13 100644 --- a/client.go +++ b/client.go @@ -1326,7 +1326,7 @@ func (t *Torrent) MergeSpec(spec *TorrentSpec) error { defer cl.unlock() t.initialPieceCheckDisabled = spec.DisableInitialPieceCheck for _, url := range spec.Webseeds { - t.addWebSeed(url) + t.addWebSeed(url, spec.EncodeWebSeedUrl) } for _, peerAddr := range spec.PeerAddrs { t.addPeer(PeerInfo{ diff --git a/spec.go b/spec.go index 332ea139..df5b06f9 100644 --- a/spec.go +++ b/spec.go @@ -36,6 +36,10 @@ type TorrentSpec struct { // Whether to allow data download or upload DisallowDataUpload bool DisallowDataDownload bool + + // Custom encoder for webseed URLs + // Leave nil to use the default (url.QueryEscape) + EncodeWebSeedUrl func(string) string } func TorrentSpecFromMagnetUri(uri string) (spec *TorrentSpec, err error) { diff --git a/torrent.go b/torrent.go index 6a75eff3..7810e9d1 100644 --- a/torrent.go +++ b/torrent.go @@ -2350,15 +2350,28 @@ func (t *Torrent) callbacks() *Callbacks { return &t.cl.config.Callbacks } -func (t *Torrent) AddWebSeeds(urls []string) { +type AddWebSeedsOptions struct { + // Custom encoder for webseed URLs + // Leave nil to use the default (url.QueryEscape) + EncodeUrl func(string) string +} + + +// Add web seeds to the torrent. +// If AddWebSeedsOptions is not nil, only the first one is used. +func (t *Torrent) AddWebSeeds(urls []string, opts ...AddWebSeedsOptions) { t.cl.lock() defer t.cl.unlock() for _, u := range urls { - t.addWebSeed(u) + if opts == nil { + t.addWebSeed(u, nil) + } else { + t.addWebSeed(u, opts[0].EncodeUrl) + } } } -func (t *Torrent) addWebSeed(url string) { +func (t *Torrent) addWebSeed(url string, encodeUrl func(string) string) { if t.cl.config.DisableWebseeds { return } @@ -2395,6 +2408,7 @@ func (t *Torrent) addWebSeed(url string) { r: r, } }, + EncodeUrl: encodeUrl, }, activeRequests: make(map[Request]webseed.Request, maxRequests), maxRequests: maxRequests, diff --git a/webseed/client.go b/webseed/client.go index a04b3430..ce8e3a42 100644 --- a/webseed/client.go +++ b/webseed/client.go @@ -41,6 +41,11 @@ func (r Request) Cancel() { r.cancel() } +type Spec struct { + Urls []string + EncodeUrl func(string) string +} + type Client struct { HttpClient *http.Client Url string @@ -52,6 +57,7 @@ type Client struct { // private in the future, if Client ever starts removing pieces. Pieces roaring.Bitmap ResponseBodyWrapper ResponseBodyWrapper + EncodeUrl func(string) string } type ResponseBodyWrapper func(io.Reader) io.Reader @@ -76,7 +82,10 @@ func (ws *Client) NewRequest(r RequestSpec) Request { ctx, cancel := context.WithCancel(context.Background()) var requestParts []requestPart if !ws.fileIndex.Locate(r, func(i int, e segments.Extent) bool { - req, err := NewRequest(ws.Url, i, ws.info, e.Start, e.Length) + req, err := NewRequestWithCustomUrlEncoding( + ws.Url, i, ws.info, e.Start, e.Length, + ws.EncodeUrl, + ) if err != nil { panic(err) } diff --git a/webseed/request.go b/webseed/request.go index 4e3ef609..3d4b4931 100644 --- a/webseed/request.go +++ b/webseed/request.go @@ -12,28 +12,56 @@ import ( // Escapes path name components suitable for appending to a webseed URL. This works for converting // S3 object keys to URLs too. +// Contrary to the name, this actually does a QueryEscape, rather than a +// PathEscape. This works better with most S3 providers. You can use +// EscapePathWithCustomEncoding for a custom encoding. func EscapePath(pathComps []string) string { + return escapePath(pathComps, nil) +} + +func escapePath(pathComps []string, encodeUrl func(string) string) string { + if encodeUrl == nil { + encodeUrl = url.QueryEscape + } return path.Join( func() (ret []string) { for _, comp := range pathComps { - ret = append(ret, url.QueryEscape(comp)) + ret = append(ret, encodeUrl(comp)) } return }()..., ) } -func trailingPath(infoName string, fileComps []string) string { - return EscapePath(append([]string{infoName}, fileComps...)) +func trailingPath(infoName string, fileComps []string, encodeUrl func(string) string) string { + return escapePath(append([]string{infoName}, fileComps...), encodeUrl) } // Creates a request per BEP 19. func NewRequest(url_ string, fileIndex int, info *metainfo.Info, offset, length int64) (*http.Request, error) { + return newRequest(url_, fileIndex, info, offset, length, nil) +} + +func NewRequestWithCustomUrlEncoding( + url_ string, fileIndex int, + info *metainfo.Info, + offset, length int64, + encodeUrl func(string) string, +) (*http.Request, error) { + return newRequest(url_, fileIndex, info, offset, length, encodeUrl) +} + +func newRequest( + url_ string, fileIndex int, + info *metainfo.Info, + offset, length int64, + encodeUrl func(string) string, +) (*http.Request, error) { fileInfo := info.UpvertedFiles()[fileIndex] if strings.HasSuffix(url_, "/") { // BEP specifies that we append the file path. We need to escape each component of the path // for things like spaces and '#'. - url_ += trailingPath(info.Name, fileInfo.Path) + url_ += trailingPath(info.Name, fileInfo.Path, encodeUrl) } req, err := http.NewRequest(http.MethodGet, url_, nil) if err != nil { diff --git a/webseed/request_test.go b/webseed/request_test.go index f7c18a03..b59e7784 100644 --- a/webseed/request_test.go +++ b/webseed/request_test.go @@ -10,7 +10,7 @@ import ( func TestTrailingPath(t *testing.T) { c := qt.New(t) test := func(parts []string, result string) { - unescaped, err := url.QueryUnescape(trailingPath(parts[0], parts[1:])) + unescaped, err := url.QueryUnescape(trailingPath(parts[0], parts[1:], url.QueryEscape)) if !c.Check(err, qt.IsNil) { return } @@ -23,7 +23,7 @@ func TestTrailingPath(t *testing.T) { } func TestTrailingPathForEmptyInfoName(t *testing.T) { - qt.Check(t, trailingPath("", []string{`ノ┬─┬ノ ︵ ( \o°o)\`}), qt.Equals, "%E3%83%8E%E2%94%AC%E2%94%80%E2%94%AC%E3%83%8E+%EF%B8%B5+%28+%5Co%C2%B0o%29%5C") - qt.Check(t, trailingPath("", []string{"hello", "world"}), qt.Equals, "hello/world") - qt.Check(t, trailingPath("war", []string{"and", "peace"}), qt.Equals, "war/and/peace") + qt.Check(t, trailingPath("", []string{`ノ┬─┬ノ ︵ ( \o°o)\`}, url.QueryEscape), qt.Equals, "%E3%83%8E%E2%94%AC%E2%94%80%E2%94%AC%E3%83%8E+%EF%B8%B5+%28+%5Co%C2%B0o%29%5C") + qt.Check(t, trailingPath("", []string{"hello", "world"}, url.QueryEscape), qt.Equals, "hello/world") + qt.Check(t, trailingPath("war", []string{"and", "peace"}, url.QueryEscape), qt.Equals, "war/and/peace") } -- 2.48.1