13 "github.com/RoaringBitmap/roaring"
14 "github.com/anacrolix/torrent/common"
15 "github.com/anacrolix/torrent/metainfo"
16 "github.com/anacrolix/torrent/segments"
19 type RequestSpec = segments.Extent
21 type requestPartResult struct {
26 type requestPart struct {
29 result chan requestPartResult
31 // Wrap http response bodies for such things as download rate limiting.
32 responseBodyWrapper ResponseBodyWrapper
37 Result chan RequestResult
40 func (r Request) Cancel() {
46 EncodeUrl func(string) string
50 HttpClient *http.Client
52 fileIndex segments.Index
54 // The pieces we can request with the Url. We're more likely to ban/block at the file-level
55 // given that's how requests are mapped to webseeds, but the torrent.Client works at the piece
56 // level. We can map our file-level adjustments to the pieces here. This probably need to be
57 // private in the future, if Client ever starts removing pieces.
59 ResponseBodyWrapper ResponseBodyWrapper
60 PathEscaper PathEscaper
63 type ResponseBodyWrapper func(io.Reader) io.Reader
65 func (me *Client) SetInfo(info *metainfo.Info) {
66 if !strings.HasSuffix(me.Url, "/") && info.IsDir() {
67 // In my experience, this is a non-conforming webseed. For example the
68 // http://ia600500.us.archive.org/1/items URLs in archive.org torrents.
71 me.fileIndex = segments.NewIndex(common.LengthIterFromUpvertedFiles(info.UpvertedFiles()))
73 me.Pieces.AddRange(0, uint64(info.NumPieces()))
76 type RequestResult struct {
81 func (ws *Client) NewRequest(r RequestSpec) Request {
82 ctx, cancel := context.WithCancel(context.Background())
83 var requestParts []requestPart
84 if !ws.fileIndex.Locate(r, func(i int, e segments.Extent) bool {
85 req, err := NewRequestWithOpts(
86 ws.Url, i, ws.info, e.Start, e.Length,
92 req = req.WithContext(ctx)
95 result: make(chan requestPartResult, 1),
97 responseBodyWrapper: ws.ResponseBodyWrapper,
101 resp, err := ws.HttpClient.Do(req)
102 part.result <- requestPartResult{
108 requestParts = append(requestParts, part)
111 panic("request out of file bounds")
115 Result: make(chan RequestResult, 1),
118 b, err := readRequestPartResponses(ctx, requestParts)
119 req.Result <- RequestResult{
127 type ErrBadResponse struct {
129 Response *http.Response
132 func (me ErrBadResponse) Error() string {
136 func recvPartResult(ctx context.Context, buf io.Writer, part requestPart) error {
137 result := <-part.result
138 // Make sure there's no further results coming, it should be a one-shot channel.
140 if result.err != nil {
143 defer result.resp.Body.Close()
144 var body io.Reader = result.resp.Body
145 if part.responseBodyWrapper != nil {
146 body = part.responseBodyWrapper(body)
148 // Prevent further accidental use
149 result.resp.Body = nil
150 if ctx.Err() != nil {
153 switch result.resp.StatusCode {
154 case http.StatusPartialContent:
155 copied, err := io.Copy(buf, body)
159 if copied != part.e.Length {
160 return fmt.Errorf("got %v bytes, expected %v", copied, part.e.Length)
164 // This number is based on
165 // https://archive.org/download/BloodyPitOfHorror/BloodyPitOfHorror.asr.srt. It seems that
166 // archive.org might be using a webserver implementation that refuses to do partial
167 // responses to small files.
168 if part.e.Start < 48<<10 {
169 if part.e.Start != 0 {
170 log.Printf("resp status ok but requested range [url=%q, range=%q]",
172 part.req.Header.Get("Range"))
174 // Instead of discarding, we could try receiving all the chunks present in the response
175 // body. I don't know how one would handle multiple chunk requests resulting in an OK
176 // response for the same file. The request algorithm might be need to be smarter for
178 discarded, _ := io.CopyN(io.Discard, body, part.e.Start)
180 log.Printf("discarded %v bytes in webseed request response part", discarded)
182 _, err := io.CopyN(buf, body, part.e.Length)
185 return ErrBadResponse{"resp status ok but requested range", result.resp}
187 case http.StatusServiceUnavailable:
190 return ErrBadResponse{
191 fmt.Sprintf("unhandled response status code (%v)", result.resp.StatusCode),
197 var ErrTooFast = errors.New("making requests too fast")
199 func readRequestPartResponses(ctx context.Context, parts []requestPart) (_ []byte, err error) {
201 for _, part := range parts {
203 err = recvPartResult(ctx, &buf, part)
205 err = fmt.Errorf("reading %q at %q: %w", part.req.URL, part.req.Header.Get("Range"), err)
209 return buf.Bytes(), err