From: Matt Joiner Date: Sun, 24 Mar 2024 10:55:30 +0000 (+1100) Subject: Pad v1 piece hashes for v2 files X-Git-Url: http://www.git.stargrave.org/?a=commitdiff_plain;h=5c818c3fb3d481288493e203535b393c06431012;p=btrtrc.git Pad v1 piece hashes for v2 files --- diff --git a/metainfo/piece.go b/metainfo/piece.go index c1c71221..c4101a6d 100644 --- a/metainfo/piece.go +++ b/metainfo/piece.go @@ -35,10 +35,26 @@ func (p Piece) Length() int64 { } return ret } - if p.i == p.Info.NumPieces()-1 { - return p.Info.TotalLength() - int64(p.i)*p.Info.PieceLength + return p.V1Length() +} + +func (p Piece) V1Length() int64 { + i := p.i + lastPiece := p.Info.NumPieces() - 1 + switch { + case 0 <= i && i < lastPiece: + return p.Info.PieceLength + case lastPiece >= 0 && i == lastPiece: + files := p.Info.UpvertedFiles() + lastFile := files[len(files)-1] + length := lastFile.TorrentOffset + lastFile.Length - int64(i)*p.Info.PieceLength + if length <= 0 || length > p.Info.PieceLength { + panic(length) + } + return length + default: + panic(i) } - return p.Info.PieceLength } func (p Piece) Offset() int64 { diff --git a/peerconn.go b/peerconn.go index 6aa8f210..df71abc9 100644 --- a/peerconn.go +++ b/peerconn.go @@ -1168,7 +1168,15 @@ func (c *PeerConn) pexEvent(t pexEventType) (_ pexEvent, err error) { } func (pc *PeerConn) String() string { - return fmt.Sprintf("%T %p [id=%+q, exts=%v, v=%q]", pc, pc, pc.PeerID, pc.PeerExtensionBytes, pc.PeerClientName.Load()) + return fmt.Sprintf( + "%T %p [flags=%v id=%+q, exts=%v, v=%q]", + pc, + pc, + pc.connectionFlags(), + pc.PeerID, + pc.PeerExtensionBytes, + pc.PeerClientName.Load(), + ) } // Returns the pieces the peer could have based on their claims. If we don't know how many pieces diff --git a/piece.go b/piece.go index c65ba9a0..73d1e0c1 100644 --- a/piece.go +++ b/piece.go @@ -11,14 +11,13 @@ import ( "github.com/anacrolix/torrent/metainfo" pp "github.com/anacrolix/torrent/peer_protocol" "github.com/anacrolix/torrent/storage" - infohash_v2 "github.com/anacrolix/torrent/types/infohash-v2" ) type Piece struct { // The completed piece SHA1 hash, from the metainfo "pieces" field. Nil if the info is not V1 // compatible. hash *metainfo.Hash - hashV2 g.Option[infohash_v2.T] + hashV2 g.Option[[32]byte] t *Torrent index pieceIndex files []*File @@ -52,7 +51,7 @@ func (p *Piece) String() string { } func (p *Piece) Info() metainfo.Piece { - return p.t.info.Piece(int(p.index)) + return p.t.info.Piece(p.index) } func (p *Piece) Storage() storage.Piece { @@ -60,7 +59,7 @@ func (p *Piece) Storage() storage.Piece { if p.hash != nil { pieceHash.Set(p.hash.Bytes()) } else if p.hashV2.Ok { - pieceHash.Set(p.hashV2.Value.Bytes()) + pieceHash.Set(p.hashV2.Value[:]) } return p.t.storage.PieceWithHash(p.Info(), pieceHash) } diff --git a/storage/file_test.go b/storage/file_test.go index a6c69fa2..aada9193 100644 --- a/storage/file_test.go +++ b/storage/file_test.go @@ -22,6 +22,7 @@ func TestShortFile(t *testing.T) { Name: "a", Length: 2, PieceLength: missinggo.MiB, + Pieces: make([]byte, 20), } ts, err := s.OpenTorrent(info, metainfo.Hash{}) assert.NoError(t, err) diff --git a/storage/issue95_test.go b/storage/issue95_test.go index 61acf17e..5b484b30 100644 --- a/storage/issue95_test.go +++ b/storage/issue95_test.go @@ -13,22 +13,19 @@ import ( // Two different torrents opened from the same storage. Closing one should not // break the piece completion on the other. func testIssue95(t *testing.T, ci ClientImpl) { - i1 := &metainfo.Info{ - Files: []metainfo.FileInfo{{Path: []string{"a"}}}, - Pieces: make([]byte, 20), + info := metainfo.Info{ + Files: []metainfo.FileInfo{{Path: []string{"a"}, Length: 1}}, + Pieces: make([]byte, 20), + PieceLength: 1, } c := NewClient(ci) - t1, err := c.OpenTorrent(i1, metainfo.HashBytes([]byte("a"))) + t1, err := c.OpenTorrent(&info, metainfo.HashBytes([]byte("a"))) require.NoError(t, err) defer t1.Close() - i2 := &metainfo.Info{ - Files: []metainfo.FileInfo{{Path: []string{"a"}}}, - Pieces: make([]byte, 20), - } - t2, err := c.OpenTorrent(i2, metainfo.HashBytes([]byte("b"))) + t2, err := c.OpenTorrent(&info, metainfo.HashBytes([]byte("b"))) require.NoError(t, err) defer t2.Close() - t2p := t2.Piece(i2.Piece(0)) + t2p := t2.Piece(info.Piece(0)) assert.NoError(t, t1.Close()) assert.NotPanics(t, func() { t2p.Completion() }) } diff --git a/storage/issue96_test.go b/storage/issue96_test.go index d9857881..cac5d96b 100644 --- a/storage/issue96_test.go +++ b/storage/issue96_test.go @@ -17,6 +17,7 @@ func testMarkedCompleteMissingOnRead(t *testing.T, csf func(string) ClientImplCl info := &metainfo.Info{ PieceLength: 1, Files: []metainfo.FileInfo{{Path: []string{"a"}, Length: 1}}, + Pieces: make([]byte, 20), } ts, err := cs.OpenTorrent(info, metainfo.Hash{}) require.NoError(t, err) diff --git a/torrent.go b/torrent.go index 39bffd77..e0ce8184 100644 --- a/torrent.go +++ b/torrent.go @@ -1153,10 +1153,7 @@ func (t *Torrent) hashPiece(piece pieceIndex) ( p.waitNoPendingWrites() storagePiece := p.Storage() - var h hash.Hash if p.hash != nil { - h = pieceHash.New() - // Does the backend want to do its own hashing? if i, ok := storagePiece.PieceImpl.(storage.SelfHashing); ok { var sum metainfo.Hash @@ -1167,12 +1164,37 @@ func (t *Torrent) hashPiece(piece pieceIndex) ( // in pieceHasher regardless. return } - + h := pieceHash.New() + differingPeers, err = t.hashPieceWithSpecificHash(piece, h, t.info.FilesArePieceAligned()) + var sum [20]byte + n := len(h.Sum(sum[:0])) + if n != 20 { + panic(n) + } + correct = sum == *p.hash } else if p.hashV2.Ok { - h = merkle.NewHash() + h := merkle.NewHash() + differingPeers, err = t.hashPieceWithSpecificHash(piece, h, false) + var sum [32]byte + n := len(h.Sum(sum[:0])) + if n != 32 { + panic(n) + } + correct = sum == p.hashV2.Value } else { panic("no hash") } + return +} + +func (t *Torrent) hashPieceWithSpecificHash(piece pieceIndex, h hash.Hash, padV1 bool) ( + // These are peers that sent us blocks that differ from what we hash here. + differingPeers map[bannableAddr]struct{}, + err error, +) { + p := t.piece(piece) + p.waitNoPendingWrites() + storagePiece := p.Storage() const logPieceContents = false smartBanWriter := t.smartBanBlockCheckingWriter(piece) @@ -1181,32 +1203,37 @@ func (t *Torrent) hashPiece(piece pieceIndex) ( if logPieceContents { writers = append(writers, &examineBuf) } - var written int64 - written, err = storagePiece.WriteTo(io.MultiWriter(writers...)) - if err == nil && written != int64(p.length()) { - err = fmt.Errorf("wrote %v bytes from storage, piece has length %v", written, p.length()) - } - if logPieceContents { - t.logger.WithDefaultLevel(log.Debug).Printf("hashed %q with copy err %v", examineBuf.Bytes(), err) + multiWriter := io.MultiWriter(writers...) + { + var written int64 + written, err = storagePiece.WriteTo(multiWriter) + if err == nil && written != int64(p.length()) { + err = fmt.Errorf("wrote %v bytes from storage, piece has length %v", written, p.length()) + // Skip smart banning since we can't blame them for storage issues. A short write would + // ban peers for all recorded blocks that weren't just written. + return + } } + // Flush before writing padding, since we would not have recorded the padding blocks. smartBanWriter.Flush() differingPeers = smartBanWriter.badPeers - if p.hash != nil { - var sum [20]byte - n := len(h.Sum(sum[:0])) - if n != 20 { - panic(n) - } - correct = sum == *p.hash - } else if p.hashV2.Ok { - var sum [32]byte - n := len(h.Sum(sum[:0])) - if n != 32 { - panic(n) + // For a hybrid torrent, we work with the v2 files, but if we use a v1 hash, we can assume that + // the pieces are padded with zeroes. + if padV1 { + paddingLen := p.Info().V1Length() - p.Info().Length() + written, err := io.CopyN(multiWriter, zeroReader, paddingLen) + if written != paddingLen { + panic(fmt.Sprintf( + "piece %v: wrote %v bytes of padding, expected %v, error: %v", + piece, + written, + paddingLen, + err, + )) } - correct = sum == p.hashV2.Value - } else { - panic("no hash") + } + if logPieceContents { + t.logger.WithNames("hashing").Levelf(log.Debug, "hashed %q with copy err %v", examineBuf.Bytes(), err) } return } @@ -2420,7 +2447,12 @@ func (t *Torrent) pieceHashed(piece pieceIndex, passed bool, hashIoErr error) { // single peer for a piece, and we never progress that piece to completion, we // will never smart-ban them. Discovered in // https://github.com/anacrolix/torrent/issues/715. - t.logger.Levelf(log.Warning, "banning %v for being sole dirtier of piece %v after failed piece check", c, piece) + t.logger.Levelf( + log.Warning, + "banning %v for being sole dirtier of piece %v after failed piece check", + c, + piece, + ) c.ban() } } @@ -2543,7 +2575,7 @@ func (t *Torrent) pieceHasher(index pieceIndex) { switch copyErr { case nil, io.EOF: default: - t.logger.Levelf( + t.logger.WithNames("hashing").Levelf( log.Warning, "error hashing piece %v: %v", index, copyErr) } diff --git a/zero-reader.go b/zero-reader.go new file mode 100644 index 00000000..1d0a899b --- /dev/null +++ b/zero-reader.go @@ -0,0 +1,11 @@ +package torrent + +var zeroReader zeroReaderType + +type zeroReaderType struct{} + +func (me zeroReaderType) Read(b []byte) (n int, err error) { + clear(b) + n = len(b) + return +}