From f9c600b264769bef07b8c075a89184452c93bf92 Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Fri, 30 Oct 2015 01:21:09 +1100 Subject: [PATCH] metainfo: Add alternative "builder" API The existing builder API is gross and heavy-handed. I won't rip it out just yet. --- cmd/torrent-create/main.go | 64 +++++++------------- internal/testutil/testutil.go | 20 ++++--- metainfo/metainfo.go | 106 ++++++++++++++++++++++++++++++++++ metainfo/metainfo_test.go | 30 ++++++++++ 4 files changed, 171 insertions(+), 49 deletions(-) diff --git a/cmd/torrent-create/main.go b/cmd/torrent-create/main.go index 0a877c25..9174c857 100644 --- a/cmd/torrent-create/main.go +++ b/cmd/torrent-create/main.go @@ -1,13 +1,15 @@ package main import ( - "flag" + "io" "log" "os" "path/filepath" - "runtime" + "strings" - torrent "github.com/anacrolix/torrent/metainfo" + "github.com/docopt/docopt-go" + + "github.com/anacrolix/torrent/metainfo" ) var ( @@ -18,50 +20,28 @@ var ( } ) -func init() { - flag.Parse() - runtime.GOMAXPROCS(runtime.NumCPU()) -} - func main() { - b := torrent.Builder{} - for _, filename := range flag.Args() { - if err := filepath.Walk(filename, func(path string, info os.FileInfo, err error) error { - if _, err := os.Stat(path); os.IsNotExist(err) { - return err - } - log.Print(path) - if info.IsDir() { - return nil - } - b.AddFile(path) - return nil - }); err != nil { - log.Print(err) - } + opts, err := docopt.Parse("Usage: torrent-create ", nil, true, "", true) + if err != nil { + panic(err) } - for _, group := range builtinAnnounceList { - b.AddAnnounceGroup(group) + root := opts[""].(string) + mi := metainfo.MetaInfo{ + AnnounceList: builtinAnnounceList, } - batch, err := b.Submit() + mi.SetDefaults() + err = mi.Info.BuildFromFilePath(root) if err != nil { log.Fatal(err) } - errs, status := batch.Start(os.Stdout, runtime.NumCPU()) - lastProgress := int64(-1) - for { - select { - case err, ok := <-errs: - if !ok || err == nil { - return - } - log.Print(err) - case bytesDone := <-status: - progress := 100 * bytesDone / batch.TotalSize() - if progress != lastProgress { - log.Printf("%d%%", progress) - lastProgress = progress - } - } + err = mi.Info.GeneratePieces(func(fi metainfo.FileInfo) (io.ReadCloser, error) { + return os.Open(filepath.Join(root, strings.Join(fi.Path, string(filepath.Separator)))) + }) + if err != nil { + log.Fatalf("error generating pieces: %s", err) + } + err = mi.Write(os.Stdout) + if err != nil { + log.Fatal(err) } } diff --git a/internal/testutil/testutil.go b/internal/testutil/testutil.go index 552f9a8e..b23ed5af 100644 --- a/internal/testutil/testutil.go +++ b/internal/testutil/testutil.go @@ -26,16 +26,22 @@ func CreateDummyTorrentData(dirName string) string { // Writes to w, a metainfo containing the file at name. func CreateMetaInfo(name string, w io.Writer) { - builder := metainfo.Builder{} - builder.AddFile(name) - builder.AddAnnounceGroup([]string{"lol://cheezburger"}) - builder.SetPieceLength(5) - batch, err := builder.Submit() + var mi metainfo.MetaInfo + mi.Info.Name = filepath.Base(name) + fi, _ := os.Stat(name) + mi.Info.Length = fi.Size() + mi.Announce = "lol://cheezburger" + mi.Info.PieceLength = 5 + err := mi.Info.GeneratePieces(func(metainfo.FileInfo) (io.ReadCloser, error) { + return os.Open(name) + }) + if err != nil { + panic(err) + } + err = mi.Write(w) if err != nil { panic(err) } - errs, _ := batch.Start(w, 1) - <-errs } // Gives a temporary directory containing the completed "greeting" torrent, diff --git a/metainfo/metainfo.go b/metainfo/metainfo.go index 47fe7de6..deb936a2 100644 --- a/metainfo/metainfo.go +++ b/metainfo/metainfo.go @@ -2,8 +2,14 @@ package metainfo import ( "crypto/sha1" + "errors" + "fmt" "io" + "log" "os" + "path/filepath" + "strings" + "time" "github.com/anacrolix/torrent/bencode" ) @@ -46,6 +52,90 @@ type Info struct { Files []FileInfo `bencode:"files,omitempty"` } +func (info *Info) BuildFromFilePath(root string) (err error) { + info.Name = filepath.Base(root) + info.Files = nil + err = filepath.Walk(root, func(path string, fi os.FileInfo, err error) error { + log.Println(path, root, err) + if fi.IsDir() { + // Directories are implicit in torrent files. + return nil + } else if path == root { + // The root is a file. + info.Length = fi.Size() + return nil + } + relPath, err := filepath.Rel(root, path) + log.Println(relPath, err) + if err != nil { + return fmt.Errorf("error getting relative path: %s", err) + } + info.Files = append(info.Files, FileInfo{ + Path: strings.Split(relPath, string(filepath.Separator)), + Length: fi.Size(), + }) + return nil + }) + if err != nil { + return + } + err = info.GeneratePieces(func(fi FileInfo) (io.ReadCloser, error) { + return os.Open(filepath.Join(root, strings.Join(fi.Path, string(filepath.Separator)))) + }) + if err != nil { + err = fmt.Errorf("error generating pieces: %s", err) + } + return +} + +func (info *Info) writeFiles(w io.Writer, open func(fi FileInfo) (io.ReadCloser, error)) error { + for _, fi := range info.UpvertedFiles() { + r, err := open(fi) + if err != nil { + return fmt.Errorf("error opening %s: %s", fi, err) + } + wn, err := io.CopyN(w, r, fi.Length) + r.Close() + if wn != fi.Length || err != nil { + return fmt.Errorf("error hashing %s: %s", fi, err) + } + } + return nil +} + +// Set info.Pieces by hashing info.Files. +func (info *Info) GeneratePieces(open func(fi FileInfo) (io.ReadCloser, error)) error { + if info.PieceLength == 0 { + return errors.New("piece length must be non-zero") + } + pr, pw := io.Pipe() + go func() { + err := info.writeFiles(pw, open) + pw.CloseWithError(err) + }() + defer pr.Close() + var pieces []byte + for { + hasher := sha1.New() + wn, err := io.CopyN(hasher, pr, info.PieceLength) + if err == io.EOF { + err = nil + } + if err != nil { + return err + } + if wn == 0 { + break + } + pieces = hasher.Sum(pieces) + if wn < info.PieceLength { + break + } + } + info.Pieces = pieces + return nil +} + func (me *Info) TotalLength() (ret int64) { if me.IsDir() { for _, fi := range me.Files { @@ -58,6 +148,9 @@ func (me *Info) TotalLength() (ret int64) { } func (me *Info) NumPieces() int { + if len(me.Pieces)%20 != 0 { + panic(len(me.Pieces)) + } return len(me.Pieces) / 20 } @@ -147,3 +240,16 @@ type MetaInfo struct { Encoding string `bencode:"encoding,omitempty"` URLList interface{} `bencode:"url-list,omitempty"` } + +// Encode to bencoded form. +func (mi *MetaInfo) Write(w io.Writer) error { + return bencode.NewEncoder(w).Encode(mi) +} + +// Set good default values in preparation for creating a new MetaInfo file. +func (mi *MetaInfo) SetDefaults() { + mi.Comment = "yoloham" + mi.CreatedBy = "github.com/anacrolix/torrent" + mi.CreationDate = time.Now().Unix() + mi.Info.PieceLength = 256 * 1024 +} diff --git a/metainfo/metainfo_test.go b/metainfo/metainfo_test.go index 16746c5b..2e6d8f08 100644 --- a/metainfo/metainfo_test.go +++ b/metainfo/metainfo_test.go @@ -2,9 +2,14 @@ package metainfo import ( "bytes" + "io" + "io/ioutil" "path" "testing" + "github.com/anacrolix/missinggo" + "github.com/stretchr/testify/assert" + "github.com/anacrolix/torrent/bencode" ) @@ -45,3 +50,28 @@ func TestFile(t *testing.T) { test_file(t, "_testdata/23516C72685E8DB0C8F15553382A927F185C4F01.torrent") test_file(t, "_testdata/trackerless.torrent") } + +// Ensure that the correct number of pieces are generated when hashing files. +func TestNumPieces(t *testing.T) { + for _, _case := range []struct { + PieceLength int64 + Files []FileInfo + NumPieces int + }{ + {256 * 1024, []FileInfo{{Length: 1024*1024 + -1}}, 4}, + {256 * 1024, []FileInfo{{Length: 1024 * 1024}}, 4}, + {256 * 1024, []FileInfo{{Length: 1024*1024 + 1}}, 5}, + {5, []FileInfo{{Length: 1}, {Length: 12}}, 3}, + {5, []FileInfo{{Length: 4}, {Length: 12}}, 4}, + } { + info := Info{ + Files: _case.Files, + PieceLength: _case.PieceLength, + } + err := info.GeneratePieces(func(fi FileInfo) (io.ReadCloser, error) { + return ioutil.NopCloser(missinggo.ZeroReader{}), nil + }) + assert.NoError(t, err) + assert.EqualValues(t, _case.NumPieces, info.NumPieces()) + } +} -- 2.44.0