cmd/torrent/main.go

   1 // Downloads torrents from the command-line.
   2 package main
   3
   4 import (
   5         "errors"
   6         "expvar"
   7         "fmt"
   8         "io"
   9         stdLog "log"
  10         "net"
  11         "net/http"
  12         "os"
  13         "os/signal"
  14         "strings"
  15         "sync"
  16         "syscall"
  17         "time"
  18
  19         "github.com/anacrolix/args"
  20         "github.com/anacrolix/envpprof"
  21         "github.com/anacrolix/log"
  22         "github.com/anacrolix/missinggo/v2"
  23         "github.com/anacrolix/tagflag"
  24         "github.com/anacrolix/torrent/bencode"
  25         "github.com/anacrolix/torrent/version"
  26         "github.com/davecgh/go-spew/spew"
  27         "github.com/dustin/go-humanize"
  28         "golang.org/x/time/rate"
  29
  30         "github.com/anacrolix/torrent"
  31         "github.com/anacrolix/torrent/iplist"
  32         "github.com/anacrolix/torrent/metainfo"
  33         "github.com/anacrolix/torrent/storage"
  34 )
  35
  36 func torrentBar(t *torrent.Torrent, pieceStates bool) {
  37         go func() {
  38                 start := time.Now()
  39                 if t.Info() == nil {
  40                         fmt.Printf("%v: getting torrent info for %q\n", time.Since(start), t.Name())
  41                         <-t.GotInfo()
  42                 }
  43                 lastStats := t.Stats()
  44                 var lastLine string
  45                 interval := 3 * time.Second
  46                 for range time.Tick(interval) {
  47                         var completedPieces, partialPieces int
  48                         psrs := t.PieceStateRuns()
  49                         for _, r := range psrs {
  50                                 if r.Complete {
  51                                         completedPieces += r.Length
  52                                 }
  53                                 if r.Partial {
  54                                         partialPieces += r.Length
  55                                 }
  56                         }
  57                         stats := t.Stats()
  58                         byteRate := int64(time.Second)
  59                         byteRate *= stats.BytesReadUsefulData.Int64() - lastStats.BytesReadUsefulData.Int64()
  60                         byteRate /= int64(interval)
  61                         line := fmt.Sprintf(
  62                                 "%v: downloading %q: %s/%s, %d/%d pieces completed (%d partial): %v/s\n",
  63                                 time.Since(start),
  64                                 t.Name(),
  65                                 humanize.Bytes(uint64(t.BytesCompleted())),
  66                                 humanize.Bytes(uint64(t.Length())),
  67                                 completedPieces,
  68                                 t.NumPieces(),
  69                                 partialPieces,
  70                                 humanize.Bytes(uint64(byteRate)),
  71                         )
  72                         if line != lastLine {
  73                                 lastLine = line
  74                                 os.Stdout.WriteString(line)
  75                         }
  76                         if pieceStates {
  77                                 fmt.Println(psrs)
  78                         }
  79                         lastStats = stats
  80                 }
  81         }()
  82 }
  83
  84 type stringAddr string
  85
  86 func (stringAddr) Network() string   { return "" }
  87 func (me stringAddr) String() string { return string(me) }
  88
  89 func resolveTestPeers(addrs []string) (ret []torrent.PeerInfo) {
  90         for _, ta := range addrs {
  91                 ret = append(ret, torrent.PeerInfo{
  92                         Addr: stringAddr(ta),
  93                 })
  94         }
  95         return
  96 }
  97
  98 func addTorrents(client *torrent.Client, flags downloadFlags) error {
  99         testPeers := resolveTestPeers(flags.TestPeer)
 100         for _, arg := range flags.Torrent {
 101                 t, err := func() (*torrent.Torrent, error) {
 102                         if strings.HasPrefix(arg, "magnet:") {
 103                                 t, err := client.AddMagnet(arg)
 104                                 if err != nil {
 105                                         return nil, fmt.Errorf("error adding magnet: %w", err)
 106                                 }
 107                                 return t, nil
 108                         } else if strings.HasPrefix(arg, "http://") || strings.HasPrefix(arg, "https://") {
 109                                 response, err := http.Get(arg)
 110                                 if err != nil {
 111                                         return nil, fmt.Errorf("Error downloading torrent file: %s", err)
 112                                 }
 113
 114                                 metaInfo, err := metainfo.Load(response.Body)
 115                                 defer response.Body.Close()
 116                                 if err != nil {
 117                                         return nil, fmt.Errorf("error loading torrent file %q: %s\n", arg, err)
 118                                 }
 119                                 t, err := client.AddTorrent(metaInfo)
 120                                 if err != nil {
 121                                         return nil, fmt.Errorf("adding torrent: %w", err)
 122                                 }
 123                                 return t, nil
 124                         } else if strings.HasPrefix(arg, "infohash:") {
 125                                 t, _ := client.AddTorrentInfoHash(metainfo.NewHashFromHex(strings.TrimPrefix(arg, "infohash:")))
 126                                 return t, nil
 127                         } else {
 128                                 metaInfo, err := metainfo.LoadFromFile(arg)
 129                                 if err != nil {
 130                                         return nil, fmt.Errorf("error loading torrent file %q: %s\n", arg, err)
 131                                 }
 132                                 t, err := client.AddTorrent(metaInfo)
 133                                 if err != nil {
 134                                         return nil, fmt.Errorf("adding torrent: %w", err)
 135                                 }
 136                                 return t, nil
 137                         }
 138                 }()
 139                 if err != nil {
 140                         return fmt.Errorf("adding torrent for %q: %w", arg, err)
 141                 }
 142                 if flags.Progress {
 143                         torrentBar(t, flags.PieceStates)
 144                 }
 145                 t.AddPeers(testPeers)
 146                 go func() {
 147                         <-t.GotInfo()
 148                         if len(flags.File) == 0 {
 149                                 t.DownloadAll()
 150                         } else {
 151                                 for _, f := range t.Files() {
 152                                         for _, fileArg := range flags.File {
 153                                                 if f.DisplayPath() == fileArg {
 154                                                         f.Download()
 155                                                 }
 156                                         }
 157                                 }
 158                         }
 159                 }()
 160         }
 161         return nil
 162 }
 163
 164 type downloadFlags struct {
 165         Debug bool
 166         DownloadCmd
 167 }
 168
 169 type DownloadCmd struct {
 170         Mmap               bool           `help:"memory-map torrent data"`
 171         TestPeer           []string       `help:"addresses of some starting peers"`
 172         Seed               bool           `help:"seed after download is complete"`
 173         Addr               string         `help:"network listen addr"`
 174         MaxUnverifiedBytes tagflag.Bytes  `help:"maximum number bytes to have pending verification"`
 175         UploadRate         *tagflag.Bytes `help:"max piece bytes to send per second"`
 176         DownloadRate       *tagflag.Bytes `help:"max bytes per second down from peers"`
 177         PackedBlocklist    string
 178         PublicIP           net.IP
 179         Progress           bool `default:"true"`
 180         PieceStates        bool
 181         Quiet              bool `help:"discard client logging"`
 182         Stats              bool `help:"print stats at termination"`
 183         Dht                bool `default:"true"`
 184
 185         TcpPeers        bool `default:"true"`
 186         UtpPeers        bool `default:"true"`
 187         Webtorrent      bool `default:"true"`
 188         DisableWebseeds bool
 189
 190         Ipv4 bool `default:"true"`
 191         Ipv6 bool `default:"true"`
 192         Pex  bool `default:"true"`
 193
 194         File    []string
 195         Torrent []string `arity:"+" help:"torrent file path or magnet uri" arg:"positional"`
 196 }
 197
 198 func statsEnabled(flags downloadFlags) bool {
 199         return flags.Stats
 200 }
 201
 202 func exitSignalHandlers(notify *missinggo.SynchronizedEvent) {
 203         c := make(chan os.Signal, 1)
 204         signal.Notify(c, syscall.SIGINT, syscall.SIGTERM)
 205         for {
 206                 log.Printf("close signal received: %+v", <-c)
 207                 notify.Set()
 208         }
 209 }
 210
 211 func main() {
 212         if err := mainErr(); err != nil {
 213                 log.Printf("error in main: %v", err)
 214                 os.Exit(1)
 215         }
 216 }
 217
 218 func mainErr() error {
 219         defer envpprof.Stop()
 220         stdLog.SetFlags(stdLog.Flags() | stdLog.Lshortfile)
 221         debug := args.Flag(args.FlagOpt{Long: "debug"})
 222         args.ParseMain(
 223                 debug,
 224                 args.Subcommand("metainfo", metainfoCmd),
 225                 args.Subcommand("announce", func(p args.SubCmdCtx) error {
 226                         var cmd AnnounceCmd
 227                         err := p.NewParser().AddParams(
 228                                 args.Pos("tracker", &cmd.Tracker),
 229                                 args.Pos("infohash", &cmd.InfoHash)).Parse()
 230                         if err != nil {
 231                                 return err
 232                         }
 233                         return announceErr(cmd)
 234                 }),
 235                 args.Subcommand("download", func(p args.SubCmdCtx) error {
 236                         var dlc DownloadCmd
 237                         err := p.NewParser().AddParams(
 238                                 append(args.FromStruct(&dlc), debug)...,
 239                         ).Parse()
 240                         if err != nil {
 241                                 return err
 242                         }
 243                         dlf := downloadFlags{
 244                                 Debug:       debug.Bool(),
 245                                 DownloadCmd: dlc,
 246                         }
 247                         p.Defer(func() error {
 248                                 return downloadErr(dlf)
 249                         })
 250                         return nil
 251                 }),
 252                 args.Subcommand(
 253                         "spew-bencoding",
 254                         func(p args.SubCmdCtx) error {
 255                                 d := bencode.NewDecoder(os.Stdin)
 256                                 for i := 0; ; i++ {
 257                                         var v interface{}
 258                                         err := d.Decode(&v)
 259                                         if err == io.EOF {
 260                                                 break
 261                                         }
 262                                         if err != nil {
 263                                                 return fmt.Errorf("decoding message index %d: %w", i, err)
 264                                         }
 265                                         spew.Dump(v)
 266                                 }
 267                                 return nil
 268                         },
 269                         args.Help("reads bencoding from stdin into Go native types and spews the result"),
 270                 ),
 271                 args.Subcommand("version", func(p args.SubCmdCtx) error {
 272                         fmt.Printf("HTTP User-Agent: %q\n", version.DefaultHttpUserAgent)
 273                         fmt.Printf("Torrent client version: %q\n", version.DefaultExtendedHandshakeClientVersion)
 274                         fmt.Printf("Torrent version prefix: %q\n", version.DefaultBep20Prefix)
 275                         return nil
 276                 }),
 277         )
 278         return nil
 279 }
 280
 281 func downloadErr(flags downloadFlags) error {
 282         clientConfig := torrent.NewDefaultClientConfig()
 283         clientConfig.DisableWebseeds = flags.DisableWebseeds
 284         clientConfig.DisableTCP = !flags.TcpPeers
 285         clientConfig.DisableUTP = !flags.UtpPeers
 286         clientConfig.DisableIPv4 = !flags.Ipv4
 287         clientConfig.DisableIPv6 = !flags.Ipv6
 288         clientConfig.DisableAcceptRateLimiting = true
 289         clientConfig.NoDHT = !flags.Dht
 290         clientConfig.Debug = flags.Debug
 291         clientConfig.Seed = flags.Seed
 292         clientConfig.PublicIp4 = flags.PublicIP
 293         clientConfig.PublicIp6 = flags.PublicIP
 294         clientConfig.DisablePEX = !flags.Pex
 295         clientConfig.DisableWebtorrent = !flags.Webtorrent
 296         if flags.PackedBlocklist != "" {
 297                 blocklist, err := iplist.MMapPackedFile(flags.PackedBlocklist)
 298                 if err != nil {
 299                         return fmt.Errorf("loading blocklist: %v", err)
 300                 }
 301                 defer blocklist.Close()
 302                 clientConfig.IPBlocklist = blocklist
 303         }
 304         if flags.Mmap {
 305                 clientConfig.DefaultStorage = storage.NewMMap("")
 306         }
 307         if flags.Addr != "" {
 308                 clientConfig.SetListenAddr(flags.Addr)
 309         }
 310         if flags.UploadRate != nil {
 311                 clientConfig.UploadRateLimiter = rate.NewLimiter(rate.Limit(*flags.UploadRate), 256<<10)
 312         }
 313         if flags.DownloadRate != nil {
 314                 clientConfig.DownloadRateLimiter = rate.NewLimiter(rate.Limit(*flags.DownloadRate), 1<<20)
 315         }
 316         if flags.Quiet {
 317                 clientConfig.Logger = log.Discard
 318         }
 319         clientConfig.MaxUnverifiedBytes = flags.MaxUnverifiedBytes.Int64()
 320
 321         var stop missinggo.SynchronizedEvent
 322         defer func() {
 323                 stop.Set()
 324         }()
 325
 326         client, err := torrent.NewClient(clientConfig)
 327         if err != nil {
 328                 return fmt.Errorf("creating client: %w", err)
 329         }
 330         var clientClose sync.Once //In certain situations, close was being called more than once.
 331         defer clientClose.Do(func() { client.Close() })
 332         go exitSignalHandlers(&stop)
 333         go func() {
 334                 <-stop.C()
 335                 clientClose.Do(func() { client.Close() })
 336         }()
 337
 338         // Write status on the root path on the default HTTP muxer. This will be bound to localhost
 339         // somewhere if GOPPROF is set, thanks to the envpprof import.
 340         http.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
 341                 client.WriteStatus(w)
 342         })
 343         err = addTorrents(client, flags)
 344         started := time.Now()
 345         if err != nil {
 346                 return fmt.Errorf("adding torrents: %w", err)
 347         }
 348         defer outputStats(client, flags)
 349         if client.WaitAll() {
 350                 log.Print("downloaded ALL the torrents")
 351         } else {
 352                 err = errors.New("y u no complete torrents?!")
 353         }
 354         clientConnStats := client.ConnStats()
 355         log.Printf("average download rate: %v",
 356                 humanize.Bytes(uint64(
 357                         time.Duration(
 358                                 clientConnStats.BytesReadUsefulData.Int64(),
 359                         )*time.Second/time.Since(started),
 360                 )))
 361         if flags.Seed {
 362                 if len(client.Torrents()) == 0 {
 363                         log.Print("no torrents to seed")
 364                 } else {
 365                         outputStats(client, flags)
 366                         <-stop.C()
 367                 }
 368         }
 369         spew.Dump(expvar.Get("torrent").(*expvar.Map).Get("chunks received"))
 370         spew.Dump(client.ConnStats())
 371         clStats := client.ConnStats()
 372         sentOverhead := clStats.BytesWritten.Int64() - clStats.BytesWrittenData.Int64()
 373         log.Printf(
 374                 "client read %v, %.1f%% was useful data. sent %v non-data bytes",
 375                 humanize.Bytes(uint64(clStats.BytesRead.Int64())),
 376                 100*float64(clStats.BytesReadUsefulData.Int64())/float64(clStats.BytesRead.Int64()),
 377                 humanize.Bytes(uint64(sentOverhead)))
 378         return err
 379 }
 380
 381 func outputStats(cl *torrent.Client, args downloadFlags) {
 382         if !statsEnabled(args) {
 383                 return
 384         }
 385         expvar.Do(func(kv expvar.KeyValue) {
 386                 fmt.Printf("%s: %s\n", kv.Key, kv.Value)
 387         })
 388         cl.WriteStatus(os.Stdout)
 389 }