README | 1 + cmd/btrtrc/README | 20 ++++++++++++++++++++ cmd/btrtrc/USAGE | 77 +++++++++++++++++++++++++++++++++++++++++++++++++++++ cmd/btrtrc/btrtrc-bf-stats | 7 +++++++ cmd/btrtrc/btrtrc-del | 8 ++++++++ cmd/btrtrc/btrtrc-seed-feed | 16 ++++++++++++++++ cmd/btrtrc/build | 1 + cmd/btrtrc/colour.go | 25 +++++++++++++++++++++++++ cmd/btrtrc/fifos.go | 452 +++++++++++++++++++++++++++++++++++++++++++++++++++++ cmd/btrtrc/main.go | 117 +++++++++++++++++++++++++++++++++++++++++++++++++++++ cmd/btrtrc/pc.go | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++++ cmd/btrtrc/sort.go | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++ cmd/btrtrc/status.go | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ cmd/btrtrc/txstats.go | 79 +++++++++++++++++++++++++++++++++++++++++++++++++++++ cmd/btrtrc/verify.go | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++++ cmd/torrent-list/main.go | 99 +++++++++++++++++++++++++++++++++++++++++++++++++++++ go.mod | 5 +++-- go.sum | 6 ++++-- peer.go | 4 ++++ peerconn.go | 14 ++++++++++---- storage/file-client.go | 22 ++++------------------ tracker_scraper.go | 6 ------ diff --git a/README b/README new file mode 120000 index 0000000000000000000000000000000000000000..a4787965604142b41146bed546285d6f602d0887 --- /dev/null +++ b/README @@ -0,0 +1 @@ +cmd/btrtrc/README \ No newline at end of file diff --git a/cmd/btrtrc/README b/cmd/btrtrc/README new file mode 100644 index 0000000000000000000000000000000000000000..3b8dcb8911db9e6e9053c2e009ecb66870abf193 --- /dev/null +++ b/cmd/btrtrc/README @@ -0,0 +1,20 @@ +btrtrc -- better BitTorrent client + +This is a fork of https://github.com/anacrolix/torrent BitTorrent +library with own cmd/btrtrc client implementation. Comparing to +cmd/torrent it has much less configuration options, mainly hardcoding +the most of them. But what advantages does it have, not including its native +capabilities (IPv6, DHT, μTP, PEX, magnet links, HTTP/UDP trackers)? + +* Ability to specify both IPv4 and IPv6 addresses to announce +* Ability to specify DHT bootstrap nodes +* Dynamic addition and removing of the torrents +* Much richer and nicer coloured status output +* Ability to calculate seed ratio, by remembering outgoing traffic amount +* Simpler piece completion database per each torrent in separate files +* Ability to sequentially create/verify torrent's piece completion database + +Look at USAGE file further. + +btrtrc is free software: see the file LICENSE in the root of the +repository for copying conditions. diff --git a/cmd/btrtrc/USAGE b/cmd/btrtrc/USAGE new file mode 100644 index 0000000000000000000000000000000000000000..ef7b9cfab2b7fa6113c7d5a15f9d29655979fedd --- /dev/null +++ b/cmd/btrtrc/USAGE @@ -0,0 +1,77 @@ +You have to specify correct -bind, -4 and -6 addresses. If -4/-6 is +empty, then it won't be announced. + +Each second the current time, overall downloaded/uploaded traffic +amount, number of active peers and current download/upload speed in +KiB/sec will be shown. + +fifos subdirectory will be created with following FIFO files: + +* fifos/add -- expects newline delimited paths to .torrent files or + magnet: links +* fifos/del -- expects torrent's info hashes to remove torrents from the + client +* fifos/list -- prints coloured output of all registered torrents in the + client. It shows: + * info hash (that you can use in all other FIFOs) + * name (name of the file or root directory as a rule) + * total size + * completion percentage + * seed ratio + * download/upload speeds in KiB/sec + * number of: total/pending/active/seeder peers + * (optionally) estimated completion time +* fifos/dht -- prints DHT server statistics +* fifos/files/HASH -- prints torrent's file list with completion ratio +* fifos/peers/HASH -- prints torrent's connected peers information: + * peer's ID + * status flags, that are concatenated string of: + i -- am interested + c -- am chocking + : + C -- uT holepunched + Tr -- tracker + I -- incoming + Hg -- DHT get_peers + Ha -- DHT announce_peer + X -- PEX + M -- direct (through magnet:) + + U -- UTP + E -- RC4 encryption + e -- header encryption + v1 + v2 + : + i -- he interested + c -- he chocking + * number of completed pieces + * download/upload speeds in KiB/sec during current session + * amount of downloaded/uploaded traffic during current session + * remote address with port + * client's name +* fifos/top-seed -- list torrents sorted by total transfer data amount + +For each torrent, corresponding .torrent file will be created. +Additional symbolic link with torrent's name will lead to HASH.torrent. +HASH.bf2 file is piece completion database. HASH.tx contains overall +outgoing payload traffic amount and it is updated each 10sec. + +If you massively add a bunch of unverified torrents, then +github.com/anacrolix/torrent will deal with them in parallel. That means +hash verification is done in random order from HDD's point of view, that +is rather slow. If you are going to seed many torrents, then it is +highly advisable to pre-verify them in advance, by using -verify option +with path to all .torrent files. + +Example usage for starting torrents seeding: + + $ cat > seedfile < fifos/add + $ cat fifos/list diff --git a/cmd/btrtrc/btrtrc-bf-stats b/cmd/btrtrc/btrtrc-bf-stats new file mode 100755 index 0000000000000000000000000000000000000000..dca4bf24054bdb224d7d185720b2c3af64d83d74 --- /dev/null +++ b/cmd/btrtrc/btrtrc-bf-stats @@ -0,0 +1,7 @@ +#!/usr/bin/env perl + +my %a; +foreach $c (split "", ) { + $a{($c eq "1") ? "1" : "0"}++; +} +print "0:$a{0} 1:$a{1}\n"; diff --git a/cmd/btrtrc/btrtrc-del b/cmd/btrtrc/btrtrc-del new file mode 100755 index 0000000000000000000000000000000000000000..61863853f754e8e84e88de5653dcf8789a9c3f82 --- /dev/null +++ b/cmd/btrtrc/btrtrc-del @@ -0,0 +1,8 @@ +#!/bin/sh -e + +[ -n "$1" ] +for fifo in fifos* ; do + echo $1 >$fifo/del + rm -fv txs${fifo#fifos}/$1.tx +done +rm -fv $1.bf2 diff --git a/cmd/btrtrc/btrtrc-seed-feed b/cmd/btrtrc/btrtrc-seed-feed new file mode 100755 index 0000000000000000000000000000000000000000..840aa4646d6ac5c13ca8bafabcaa74d65ce15242 --- /dev/null +++ b/cmd/btrtrc/btrtrc-seed-feed @@ -0,0 +1,16 @@ +#!/bin/sh -e +# Feed seed-file if daemon is restarted + +[ -n "$1" ] +[ -n "$2" ] +fifos=$1 +seed=$2 +while : ; do + cur=`stat -f %m $fifos/add || :` + if [ "$cur" != "$prev" ] ; then + date + grep -v "^#" $seed >$fifos/add + prev=`stat -f %m $fifos/add || :` + fi + sleep 10 +done diff --git a/cmd/btrtrc/build b/cmd/btrtrc/build new file mode 100755 index 0000000000000000000000000000000000000000..5b4238baa905e6bff828864be877fbb519f79246 --- /dev/null +++ b/cmd/btrtrc/build @@ -0,0 +1 @@ +go build -ldflags=-s -tags nosqlite,noboltdb diff --git a/cmd/btrtrc/colour.go b/cmd/btrtrc/colour.go new file mode 100644 index 0000000000000000000000000000000000000000..eb664e1ab980ff5c501bf023be33f392a7533f7a --- /dev/null +++ b/cmd/btrtrc/colour.go @@ -0,0 +1,25 @@ +package main + +import ( + "bytes" + + "golang.org/x/term" +) + +var ( + Blue string + Green string + Magenta string + Red string + Reset string +) + +func init() { + var b bytes.Buffer + t := term.NewTerminal(&b, "") + Blue = string(t.Escape.Blue) + Green = string(t.Escape.Green) + Magenta = string(t.Escape.Magenta) + Red = string(t.Escape.Red) + Reset = string(t.Escape.Reset) +} diff --git a/cmd/btrtrc/fifos.go b/cmd/btrtrc/fifos.go new file mode 100644 index 0000000000000000000000000000000000000000..d14970025450ebe378e53c7d66a0a6171b2cba0f --- /dev/null +++ b/cmd/btrtrc/fifos.go @@ -0,0 +1,452 @@ +package main + +import ( + "bufio" + "bytes" + "encoding/hex" + "fmt" + "log" + "os" + "path" + "sort" + "strconv" + "strings" + "sync" + "syscall" + "time" + + "github.com/anacrolix/dht/v2" + "github.com/anacrolix/torrent" + "github.com/anacrolix/torrent/metainfo" + "github.com/anacrolix/torrent/types/infohash" + "github.com/dustin/go-humanize" +) + +const ( + MaxListNameWidth = 40 + PeersDir = "peers" + FilesDir = "files" +) + +type TorrentStat struct { + stats torrent.AllConnStats + rxSpeed int64 + txSpeed int64 +} + +var ( + FIFOsDir = "fifos" + TorrentStats = map[metainfo.Hash]TorrentStat{} + TorrentStatsM sync.RWMutex + Torrents []*torrent.Torrent + TorrentsM sync.RWMutex +) + +func recreateFIFO(pth string) { + os.Remove(pth) + if err := syscall.Mkfifo(pth, 0o666); err != nil { + log.Fatalln(err) + } +} + +func shortenName(name string) string { + s := []rune(name) + if len(s) > MaxListNameWidth { + s = s[:MaxListNameWidth] + } + return string(s) +} + +func fifoList(c *torrent.Client) { + pth := path.Join(FIFOsDir, "list") + recreateFIFO(pth) + for { + fd, err := os.OpenFile(pth, os.O_WRONLY|os.O_APPEND, os.FileMode(0o666)) + if err != nil { + log.Println("OpenFile:", pth, err) + time.Sleep(time.Second) + continue + } + ts := make([]*torrent.Torrent, 0, len(Torrents)) + TorrentsM.RLock() + for i := range Torrents { + t := Torrents[i] + if t.Info() == nil { + fmt.Fprintf(fd, "%s not ready\n", t.Name()) + continue + } + ts = append(ts, t) + } + TorrentsM.RUnlock() + for _, t := range ts { + stats := t.Stats() + done := t.BytesCompleted() * 100 / t.Length() + percColour := Red + if done == 100 { + percColour = Green + } + tx := stats.BytesWrittenData.Int64() + tx += TxStats[t.InfoHash()] + ratio := float64(tx) / float64(t.Length()) + TorrentStatsM.RLock() + prev := TorrentStats[t.InfoHash()] + TorrentStatsM.RUnlock() + var eta string + if done < 100 && prev.rxSpeed > 0 { + etaRaw := time.Duration((t.Length() - t.BytesCompleted()) / prev.rxSpeed) + etaRaw *= time.Second + eta = etaRaw.String() + } + fmt.Fprintf(fd, + "%s%s%s %s%40s%s %8s %s%3d%%%s %4.1f %s%d%s/%s%d%s %d/%d/%d/%d %s\n", + Blue, t.InfoHash().HexString(), Reset, + Green, shortenName(t.Name()), Reset, + humanize.IBytes(uint64(t.Length())), + percColour, done, Reset, + ratio, + Green, prev.rxSpeed/1024, Reset, + Magenta, prev.txSpeed/1024, Reset, + stats.TotalPeers, + stats.PendingPeers, + stats.ActivePeers, + stats.ConnectedSeeders, + eta, + ) + } + fd.Close() + time.Sleep(time.Second) + } +} + +func mustParseInt(s string) int { + i, err := strconv.Atoi(s) + if err != nil { + log.Fatalln(err) + } + return i +} + +func fifoPeerList(t *torrent.Torrent) { + pth := path.Join(FIFOsDir, PeersDir, t.InfoHash().HexString()) + recreateFIFO(pth) + for { + fd, err := os.OpenFile(pth, os.O_WRONLY|os.O_APPEND, os.FileMode(0o666)) + if err != nil { + if os.IsNotExist(err) { + break + } + log.Println("OpenFile:", pth, err) + time.Sleep(time.Second) + continue + } + pcs := t.PeerConns() + sort.Sort(ByPeerID(pcs)) + for _, pc := range pcs { + cols := strings.Split(pc.CompletedString(), "/") + done := (mustParseInt(cols[0]) * 100) / mustParseInt(cols[1]) + doneColour := Red + if done == 100 { + doneColour = Green + } + stats := pc.Peer.Stats() + fmt.Fprintf(fd, + "%s%s%s %10s %s%3d%%%s %s%d%s/%s%d%s | %s%s%s / %s%s%s | %s%s%s %q\n", + Blue, hex.EncodeToString(pc.PeerID[:]), Reset, + pc.StatusFlags(), + doneColour, done, Reset, + Green, int(stats.DownloadRate/1024), Reset, + Magenta, int(pc.UploadRate()/1024), Reset, + Green, humanize.IBytes(uint64(stats.BytesReadData.Int64())), Reset, + Magenta, humanize.IBytes(uint64(stats.BytesWrittenData.Int64())), Reset, + Green, pc.RemoteAddr, Reset, + pc.PeerClientName, + ) + } + fd.Close() + time.Sleep(time.Second) + } +} + +func fifoFileList(t *torrent.Torrent) { + pth := path.Join(FIFOsDir, FilesDir, t.InfoHash().HexString()) + recreateFIFO(pth) + for { + fd, err := os.OpenFile(pth, os.O_WRONLY|os.O_APPEND, os.FileMode(0o666)) + if err != nil { + if os.IsNotExist(err) { + break + } + log.Println("OpenFile:", pth, err) + time.Sleep(time.Second) + continue + } + for n, f := range t.Files() { + var done int64 + if f.Length() > 0 { + done = (f.BytesCompleted() * 100) / f.Length() + } + percColour := Green + if done < 100 { + percColour = Red + } + fmt.Fprintf(fd, + "%5d %8s %3d%% | %s%s%s\n", + n, humanize.IBytes(uint64(f.Length())), done, + percColour, f.Path(), Reset, + ) + } + fd.Close() + time.Sleep(time.Second) + } +} + +func fifoDHTList(c *torrent.Client) { + pth := path.Join(FIFOsDir, "dht") + recreateFIFO(pth) + for { + fd, err := os.OpenFile(pth, os.O_WRONLY|os.O_APPEND, os.FileMode(0o666)) + if err != nil { + if os.IsNotExist(err) { + break + } + log.Println("OpenFile:", pth, err) + time.Sleep(time.Second) + continue + } + for _, s := range c.DhtServers() { + stats := s.Stats().(dht.ServerStats) + fmt.Fprintf( + fd, "%s%s%s all:%d good:%d await:%d succ:%d bad:%d\n", + Green, s.Addr().String(), Reset, + stats.Nodes, + stats.GoodNodes, + stats.OutstandingTransactions, + stats.SuccessfulOutboundAnnouncePeerQueries, + stats.BadNodes, + ) + } + fd.Close() + time.Sleep(time.Second) + } +} + +type topTorrent struct { + name string + ratio float64 + tx int64 + infoHash metainfo.Hash +} + +func fifoTopSeed(c *torrent.Client) { + pth := path.Join(FIFOsDir, "top-seed") + recreateFIFO(pth) + for { + fd, err := os.OpenFile(pth, os.O_WRONLY|os.O_APPEND, os.FileMode(0o666)) + if err != nil { + log.Println("OpenFile:", pth, err) + time.Sleep(time.Second) + continue + } + var ts []*topTorrent + for _, t := range c.Torrents() { + if t.Info() == nil { + continue + } + stats := t.Stats() + top := topTorrent{ + infoHash: t.InfoHash(), + name: t.Name(), + tx: stats.BytesWrittenData.Int64() + TxStats[t.InfoHash()], + } + top.ratio = float64(top.tx) / float64(t.Length()) + ts = append(ts, &top) + } + sort.Sort(ByTxTraffic(ts)) + for _, t := range ts { + fmt.Fprintf(fd, + "%s%s%s %s%40s%s %s%4.1f%s %s\n", + Blue, t.infoHash.HexString(), Reset, + Green, shortenName(t.name), Reset, + Magenta, t.ratio, Reset, + humanize.IBytes(uint64(t.tx)), + ) + } + fd.Close() + time.Sleep(time.Second) + } +} + +func fifoStatus(c *torrent.Client) { + pth := path.Join(FIFOsDir, "status") + recreateFIFO(pth) + for { + fd, err := os.OpenFile(pth, os.O_WRONLY|os.O_APPEND, os.FileMode(0o666)) + if err != nil { + log.Println("OpenFile:", pth, err) + time.Sleep(time.Second) + continue + } + c.WriteStatus(fd) + fd.Close() + time.Sleep(time.Second) + } +} + +type stringAddr string + +func (stringAddr) Network() string { return "" } +func (me stringAddr) String() string { return string(me) } + +func resolveTestPeers(addrs []string) (ret []torrent.PeerInfo) { + for _, ta := range addrs { + ret = append(ret, torrent.PeerInfo{Addr: stringAddr(ta)}) + } + return +} + +func readLinesFromFIFO(pth string) []string { + fd, err := os.OpenFile(pth, os.O_RDONLY, os.FileMode(0o666)) + if err != nil { + log.Println("OpenFile:", pth, err) + time.Sleep(time.Second) + return nil + } + var lines []string + scanner := bufio.NewScanner(fd) + for scanner.Scan() { + t := scanner.Text() + if len(t) > 0 { + lines = append(lines, t) + } + } + fd.Close() + return lines +} + +func saveTorrent(t *torrent.Torrent) error { + pth := t.Name() + TorrentExt + if _, err := os.Stat(pth); err == nil { + return nil + } + var b bytes.Buffer + metainfo := t.Metainfo() + metainfo.Write(&b) + return os.WriteFile(pth, b.Bytes(), 0o666) +} + +func fifoAdd(c *torrent.Client) { + pth := path.Join(FIFOsDir, "add") + recreateFIFO(pth) + for { + for _, what := range readLinesFromFIFO(pth) { + cols := strings.Split(what, "|") + what = cols[0] + var t *torrent.Torrent + var err error + if strings.HasPrefix(what, "magnet:") { + t, err = c.AddMagnet(what) + if err != nil { + log.Println("AddMagnet:", what, err) + continue + } + } else { + metaInfo, err := metainfo.LoadFromFile(what) + if err != nil { + log.Println("LoadFromFile:", what, err) + continue + } + t, err = c.AddTorrent(metaInfo) + if err != nil { + log.Println("AddTorrent:", what, err) + continue + } + } + if len(cols) > 1 { + t.AddPeers(resolveTestPeers(cols[1:])) + } + TorrentsM.Lock() + for _, _t := range Torrents { + if _t.InfoHash().HexString() == t.InfoHash().HexString() { + goto OldOne + } + } + Torrents = append(Torrents, t) + OldOne: + TorrentsM.Unlock() + go fifoPeerList(t) + go fifoFileList(t) + log.Println("added:", t.InfoHash().HexString(), t.Name()) + go func() { + <-t.GotInfo() + if err = saveTorrent(t); err != nil { + log.Println("saveTorrent:", err) + } + txStatsLoad(t.InfoHash()) + t.DownloadAll() + }() + } + time.Sleep(time.Second) + } +} + +func fifoDel(c *torrent.Client) { + pth := path.Join(FIFOsDir, "del") + recreateFIFO(pth) + for { + for _, what := range readLinesFromFIFO(pth) { + raw, err := hex.DecodeString(what) + if err != nil { + log.Println(err) + continue + } + if len(raw) != infohash.Size { + log.Println("bad length") + continue + } + var i infohash.T + copy(i[:], raw) + TorrentsM.Lock() + for n, t := range Torrents { + if t.InfoHash().HexString() == i.HexString() { + Torrents = append(Torrents[:n], Torrents[n+1:]...) + break + } + } + TorrentsM.Unlock() + t, ok := c.Torrent(i) + if !ok { + log.Println("no such torrent", what) + continue + } + txStatsDump(t) + txStatsDel(t.InfoHash()) + t.Drop() + for _, where := range []string{"files", "peers"} { + pth := path.Join(where, t.InfoHash().HexString()) + os.Remove(pth) + fd, err := os.Open(pth) + if err == nil { + fd.Close() + } + } + log.Println("deleted:", what, t.Name()) + } + time.Sleep(time.Second) + } +} + +func fifosPrepare() { + os.MkdirAll(path.Join(FIFOsDir, PeersDir), 0o777) + os.MkdirAll(path.Join(FIFOsDir, FilesDir), 0o777) +} + +func fifosCleanup() { + os.Remove(path.Join(FIFOsDir, "list")) + os.Remove(path.Join(FIFOsDir, "dht")) + os.Remove(path.Join(FIFOsDir, "add")) + os.Remove(path.Join(FIFOsDir, "del")) + os.Remove(path.Join(FIFOsDir, "top-seed")) + os.Remove(path.Join(FIFOsDir, "status")) + os.RemoveAll(path.Join(FIFOsDir, PeersDir)) + os.RemoveAll(path.Join(FIFOsDir, FilesDir)) +} diff --git a/cmd/btrtrc/main.go b/cmd/btrtrc/main.go new file mode 100644 index 0000000000000000000000000000000000000000..4640b3e94bef98dd7223d2a83e7d7114b184fe8d --- /dev/null +++ b/cmd/btrtrc/main.go @@ -0,0 +1,117 @@ +package main + +import ( + "flag" + "log" + "net" + "os" + "os/signal" + "strings" + "sync" + "syscall" + + "github.com/anacrolix/dht/v2" + analog "github.com/anacrolix/log" + "golang.org/x/time/rate" + + "github.com/anacrolix/torrent" + "github.com/anacrolix/torrent/storage" +) + +const ( + TorrentExt = ".torrent" + UserAgent = "btrtrc/0.3.0" +) + +var ( + Cancel = make(chan struct{}) + Jobs sync.WaitGroup +) + +func main() { + log.SetFlags(log.Ldate | log.Ltime) + fifosDir := flag.String("fifos", "fifos", "Path to fifos/") + txsDir := flag.String("txs", "txs", "Path to txs/") + dhtBoot := flag.String("dht", "dht.stargrave.org:8991", + "Comma-separated list of DHT bootstrap nodes") + addr := flag.String("bind", "[::]:6881", "Address to bind to") + pub4 := flag.String("4", "", "External IPv4 address") + pub6 := flag.String("6", "", "External IPv6 address") + debug := flag.Bool("debug", false, "Enable debug messages") + noDHT := flag.Bool("nodht", false, "Disable DHT") + noUTP := flag.Bool("noutp", false, "Disable uTP") + noWebseed := flag.Bool("nowebseed", false, "Disable webseeds") + rxRate := flag.Int("rx-rate", 0, "Download rate, piece bytes/sec") + txRate := flag.Int("tx-rate", 0, "Upload rate, bytes/sec") + verify := flag.Bool("verify", false, "Force verification of provided torrents") + flag.Parse() + + FIFOsDir = *fifosDir + TxsDir = *txsDir + os.MkdirAll(TxsDir, 0o777) + dht.DefaultGlobalBootstrapHostPorts = strings.Split(*dhtBoot, ",") + cc := torrent.NewDefaultClientConfig() + cc.Debug = *debug + cc.NoDefaultPortForwarding = true + cc.DisableWebtorrent = true + cc.Logger = analog.Default.WithNames("main", "client") + cc.HTTPUserAgent = UserAgent + cc.ExtendedHandshakeClientVersion = UserAgent + cc.DefaultStorage = storage.NewFileWithCompletion(".", NewBFPieceCompletion()) + cc.MaxAllocPeerRequestDataPerConn = 1 << 24 + if *verify { + doVerify(cc, flag.Args()) + return + } + cc.Seed = true + if *pub4 == "" { + cc.DisableIPv4 = true + } else { + cc.PublicIp4 = net.ParseIP(*pub4).To4() + } + if *pub6 == "" { + cc.DisableIPv6 = true + } else { + cc.PublicIp6 = net.ParseIP(*pub6).To16() + } + if *txRate != 0 { + cc.UploadRateLimiter = rate.NewLimiter(rate.Limit(*txRate), 256<<10) + } + if *rxRate != 0 { + cc.DownloadRateLimiter = rate.NewLimiter(rate.Limit(*rxRate), 1<<16) + } + cc.NoDHT = *noDHT + cc.DisableUTP = *noUTP + cc.DisableWebseeds = *noWebseed + cc.SetListenAddr(*addr) + client, err := torrent.NewClient(cc) + if err != nil { + log.Fatalln("torrent.NewClient:", err) + } + defer client.Close() + + needsShutdown := make(chan os.Signal, 1) + signal.Notify(needsShutdown, syscall.SIGTERM, syscall.SIGINT) + go func() { + <-needsShutdown + close(Cancel) + client.Close() + }() + + fifosPrepare() + log.Println("started", client.PublicIPs()) + Jobs.Add(1) + go overallStatus(client) + go fifoList(client) + go fifoTopSeed(client) + go fifoDHTList(client) + go fifoAdd(client) + go fifoDel(client) + go fifoStatus(client) + Jobs.Add(1) + go txStatsDumper(client) + <-client.Closed() + Jobs.Wait() + fifosCleanup() + log.Println("finished") +} diff --git a/cmd/btrtrc/pc.go b/cmd/btrtrc/pc.go new file mode 100644 index 0000000000000000000000000000000000000000..f1b53e7bfc99141908b655635c5e14bfa87eb128 --- /dev/null +++ b/cmd/btrtrc/pc.go @@ -0,0 +1,61 @@ +package main + +import ( + "os" + "sync" + + "github.com/anacrolix/torrent/metainfo" + "github.com/anacrolix/torrent/storage" +) + +const BFExt = ".bf2" + +type BFPieceCompletion struct { + sync.Mutex +} + +func NewBFPieceCompletion() *BFPieceCompletion { + return &BFPieceCompletion{} +} + +func (self *BFPieceCompletion) Get(pk metainfo.PieceKey) (c storage.Completion, rerr error) { + self.Lock() + defer self.Unlock() + fd, err := os.OpenFile(pk.InfoHash.HexString()+BFExt, os.O_RDWR|os.O_CREATE, 0o666) + if err != nil { + rerr = err + return + } + defer fd.Close() + b := []byte{0} + _, err = fd.ReadAt(b, int64(pk.Index)) + if err != nil { + return + } + c.Ok = true + c.Complete = b[0] == '1' + return +} + +func (self *BFPieceCompletion) Set(pk metainfo.PieceKey, complete bool) error { + if c, err := self.Get(pk); err == nil && c.Ok && c.Complete == complete { + return nil + } + self.Lock() + defer self.Unlock() + fd, err := os.OpenFile(pk.InfoHash.HexString()+BFExt, os.O_RDWR|os.O_CREATE, 0o666) + if err != nil { + return err + } + defer fd.Close() + b := []byte{'1'} + if !complete { + b[0] = 0 + } + _, err = fd.WriteAt(b, int64(pk.Index)) + return err +} + +func (self *BFPieceCompletion) Close() error { + return nil +} diff --git a/cmd/btrtrc/sort.go b/cmd/btrtrc/sort.go new file mode 100644 index 0000000000000000000000000000000000000000..3837d1f3a237754dc9b36360af452f2df8c56639 --- /dev/null +++ b/cmd/btrtrc/sort.go @@ -0,0 +1,52 @@ +package main + +import ( + "encoding/hex" + + "github.com/anacrolix/torrent" +) + +type ByInfoHash []*torrent.Torrent + +func (a ByInfoHash) Len() int { + return len(a) +} + +func (a ByInfoHash) Swap(i, j int) { + a[i], a[j] = a[j], a[i] +} + +func (a ByInfoHash) Less(i, j int) bool { + return a[i].InfoHash().HexString() < a[j].InfoHash().HexString() +} + +type ByTxTraffic []*topTorrent + +func (a ByTxTraffic) Len() int { + return len(a) +} + +func (a ByTxTraffic) Swap(i, j int) { + a[i], a[j] = a[j], a[i] +} + +func (a ByTxTraffic) Less(i, j int) bool { + if a[i].tx == a[j].tx { + return a[i].infoHash.HexString() < a[j].infoHash.HexString() + } + return a[i].tx < a[j].tx +} + +type ByPeerID []*torrent.PeerConn + +func (a ByPeerID) Len() int { + return len(a) +} + +func (a ByPeerID) Swap(i, j int) { + a[i], a[j] = a[j], a[i] +} + +func (a ByPeerID) Less(i, j int) bool { + return hex.EncodeToString(a[i].PeerID[:]) < hex.EncodeToString(a[j].PeerID[:]) +} diff --git a/cmd/btrtrc/status.go b/cmd/btrtrc/status.go new file mode 100644 index 0000000000000000000000000000000000000000..8cb0cbc479146caea5ba1421980498fb9f2de329 --- /dev/null +++ b/cmd/btrtrc/status.go @@ -0,0 +1,49 @@ +package main + +import ( + "log" + "time" + + "github.com/anacrolix/torrent" + "github.com/dustin/go-humanize" +) + +func overallStatus(c *torrent.Client) { + tick := time.Tick(time.Second) + var prev torrent.ConnStats + for { + select { + case <-Cancel: + Jobs.Done() + return + case <-tick: + } + stats := c.ConnStats() + var peers int + for _, t := range c.Torrents() { + if t.Info() == nil { + continue + } + tStats := t.Stats() + cur := tStats.Copy() + TorrentStatsM.Lock() + prev := TorrentStats[t.InfoHash()].stats + TorrentStats[t.InfoHash()] = TorrentStat{ + stats: cur, + rxSpeed: cur.BytesReadData.Int64() - prev.BytesReadData.Int64(), + txSpeed: cur.BytesWrittenData.Int64() - prev.BytesWrittenData.Int64(), + } + TorrentStatsM.Unlock() + peers += tStats.ActivePeers + } + log.Printf( + "%s / %s | %d | %s%d%s / %s%d%s", + humanize.IBytes(uint64(stats.BytesRead.Int64())), + humanize.IBytes(uint64(stats.BytesWritten.Int64())), + peers, + Green, (stats.BytesRead.Int64()-prev.BytesRead.Int64())/1024, Reset, + Magenta, (stats.BytesWritten.Int64()-prev.BytesWritten.Int64())/1024, Reset, + ) + prev = stats + } +} diff --git a/cmd/btrtrc/txstats.go b/cmd/btrtrc/txstats.go new file mode 100644 index 0000000000000000000000000000000000000000..91e52b33895aecc9858fe3d5894fa5bfab1b8a8e --- /dev/null +++ b/cmd/btrtrc/txstats.go @@ -0,0 +1,79 @@ +package main + +import ( + "fmt" + "log" + "os" + "path" + "strconv" + "sync" + "time" + + "github.com/anacrolix/torrent" + "github.com/anacrolix/torrent/metainfo" +) + +const TxExt = ".tx" + +var ( + TxStats = map[metainfo.Hash]int64{} + TxStatsM sync.Mutex + TxsDir string +) + +func txStatsLoad(h metainfo.Hash) { + pth := path.Join(TxsDir, h.HexString()+TxExt) + data, err := os.ReadFile(pth) + if err != nil { + return + } + if len(data) == 0 { + return + } + v, err := strconv.ParseInt(string(data[:len(data)-1]), 10, 64) + if err != nil { + log.Println("ParseInt:", pth, err) + return + } + TxStatsM.Lock() + TxStats[h] = v + TxStatsM.Unlock() +} + +func txStatsDel(h metainfo.Hash) { + TxStatsM.Lock() + delete(TxStats, h) + TxStatsM.Unlock() +} + +func txStatsDump(t *torrent.Torrent) { + stats := t.Stats() + TxStatsM.Lock() + s := stats.BytesWrittenData.Int64() + TxStats[t.InfoHash()] + pth := path.Join(TxsDir, t.InfoHash().HexString()+TxExt) + if err := os.WriteFile(pth, []byte(fmt.Sprintf("%d\n", s)), 0o666); err != nil { + log.Println("WriteFile:", pth, err) + } + TxStatsM.Unlock() +} + +func txStatsDumpAll(c *torrent.Client) { + for _, t := range c.Torrents() { + if t.Info() != nil { + txStatsDump(t) + } + } +} + +func txStatsDumper(c *torrent.Client) { + tick := time.Tick(10 * time.Second) + for { + txStatsDumpAll(c) + select { + case <-Cancel: + Jobs.Done() + return + case <-tick: + } + } +} diff --git a/cmd/btrtrc/verify.go b/cmd/btrtrc/verify.go new file mode 100644 index 0000000000000000000000000000000000000000..7c688ec6a7dcbe68f424943faf8e015c61acb568 --- /dev/null +++ b/cmd/btrtrc/verify.go @@ -0,0 +1,55 @@ +package main + +import ( + "context" + "fmt" + "log" + + "github.com/anacrolix/torrent" + "github.com/anacrolix/torrent/metainfo" +) + +func doVerify(cc *torrent.ClientConfig, pths []string) { + cc.DisableTrackers = true + cc.NoDHT = true + cc.NoUpload = true + cc.DisableUTP = true + cc.DisableTCP = true + cc.DisableIPv6 = true + cc.DisableIPv4 = true + cc.AcceptPeerConnections = false + cc.DisableWebseeds = true + client, err := torrent.NewClient(cc) + if err != nil { + log.Fatalln("torrent.NewClient:", err) + } + for _, pth := range pths { + metaInfo, err := metainfo.LoadFromFile(pth) + if err != nil { + log.Fatalln("LoadFromFile:", err) + } + t, err := client.AddTorrent(metaInfo) + if err != nil { + log.Fatalln("AddTorrent:", err) + } + <-t.GotInfo() + if err = saveTorrent(t); err != nil { + log.Println("saveTorrent:", err) + } + go func() { + sub := t.SubscribePieceStateChanges() + defer sub.Close() + var last int + for piece := range sub.Values { + if piece.Hashing && piece.Index > last { + fmt.Printf("\r%s: %d / %d", pth, piece.Index, t.NumPieces()) + last = piece.Index + } + } + }() + t.VerifyDataContext(context.Background()) + fmt.Printf("\n") + } + // client.Close() + // <-client.Closed() +} diff --git a/cmd/torrent-list/main.go b/cmd/torrent-list/main.go new file mode 100644 index 0000000000000000000000000000000000000000..164cf9d7915fbc5b20a6dd594a169baacf4dd375 --- /dev/null +++ b/cmd/torrent-list/main.go @@ -0,0 +1,99 @@ +package main + +import ( + "errors" + "flag" + "fmt" + "io" + "log" + "os" + "path" + + "github.com/anacrolix/torrent/bencode" + "github.com/anacrolix/torrent/metainfo" +) + +func walk(root string) ([]string, error) { + d, err := os.Open(root) + if err != nil { + return nil, err + } + paths := make([]string, 0) + for { + entries, err := d.ReadDir(1 << 10) + if err != nil { + if errors.Is(err, io.EOF) { + break + } + return paths, err + } + for _, entry := range entries { + if entry.IsDir() { + add, err := walk(path.Join(root, entry.Name())) + if err != nil { + return paths, err + } + paths = append(paths, add...) + } else { + paths = append(paths, path.Join(root, entry.Name())) + } + } + } + return paths, nil +} + +func main() { + showExcess := flag.Bool("excess", false, "Show excess files") + flag.Parse() + data, err := os.ReadFile(flag.Arg(0)) + if err != nil { + log.Fatalln(err) + } + var minfo metainfo.MetaInfo + err = bencode.Unmarshal(data, &minfo) + if err != nil { + log.Fatalln(err) + } + var info metainfo.Info + err = bencode.Unmarshal(minfo.InfoBytes, &info) + if err != nil { + log.Fatalln(err) + } + seen := make(map[string]struct{}, len(info.Files)+1) + seen[info.Name] = struct{}{} + for _, f := range info.Files { + var p string + if len(f.PathUtf8) > 0 { + p = path.Join(f.PathUtf8...) + } else { + p = path.Join(f.Path...) + } + p = path.Join(info.Name, p) + seen[p] = struct{}{} + if !*showExcess { + fmt.Print(p, "\t") + s, err := os.Stat(p) + if err != nil { + fmt.Print("!exists\n") + continue + } + if s.Size() != f.Length { + fmt.Print("!size\n") + continue + } + fmt.Print("ok\n") + } + } + if !*showExcess { + return + } + paths, err := walk(info.Name) + if err != nil { + log.Fatalln(err) + } + for _, f := range paths { + if _, exists := seen[f]; !exists { + fmt.Print(f, "\n") + } + } +} diff --git a/go.mod b/go.mod index 5ca09539f56ab4a047b5e1c55db94dcc26c65e20..47dbce7d1f5f5ec169918b68d11f9c0894e1d7bc 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/anacrolix/torrent -go 1.24 +go 1.25.0 require ( github.com/RoaringBitmap/roaring v1.2.3 @@ -55,7 +55,8 @@ golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 golang.org/x/net v0.42.0 golang.org/x/sync v0.16.0 golang.org/x/sys v0.34.0 - golang.org/x/time v0.0.0-20220609170525-579cf78fd858 + golang.org/x/term v0.33.0 + golang.org/x/time v0.15.0 ) require ( diff --git a/go.sum b/go.sum index 3e61826c5345acba1e31872b354c3e953d5eee31..9db12670f0ca772fcbaa375d3b7df080c3a9b72d 100644 --- a/go.sum +++ b/go.sum @@ -773,6 +773,8 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg= +golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -789,8 +791,8 @@ golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220609170525-579cf78fd858 h1:Dpdu/EMxGMFgq0CeYMh4fazTD2vtlZRYE7wyynxJb9U= -golang.org/x/time v0.0.0-20220609170525-579cf78fd858/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/peer.go b/peer.go index 251be54a1b0556ea0802482583b542e21aae171e..a7ad3b49874ebe59e47b580b220301407be0643d 100644 --- a/peer.go +++ b/peer.go @@ -184,6 +184,10 @@ func (cn *Peer) completedString() string { return fmt.Sprintf("%d/%d", cn.remotePieceCount(), cn.bestPeerNumPieces()) } +func (cn *Peer) CompletedString() string { + return cn.completedString() +} + func eventAgeString(t time.Time) string { if t.IsZero() { return "never" diff --git a/peerconn.go b/peerconn.go index 573ae30ed47564698a278e29c4b605882b29dc05..c7b3f97027af6849203a35b7b648b5caf584087b 100644 --- a/peerconn.go +++ b/peerconn.go @@ -129,6 +129,12 @@ defer cn.messageWriter.mu.Unlock() return cn.messageWriter.dataUploadRate } +func (cn *PeerConn) UploadRate() float64 { + cn.messageWriter.mu.Lock() + defer cn.messageWriter.mu.Unlock() + return cn.messageWriter.dataUploadRate +} + func (cn *PeerConn) pexStatus() string { if !cn.bitExtensionEnabled(pp.ExtensionBitLtep) { return "extended protocol disabled" @@ -1109,10 +1115,6 @@ } if !c.peerHasWantedPieces() { return false } - // Don't upload more than 100 KiB more than we download. - if c._stats.BytesWrittenData.Int64() >= c._stats.BytesReadData.Int64()+100<<10 { - return false - } return true } @@ -1573,6 +1575,10 @@ func (cn *PeerConn) supportsExtension(ext pp.ExtensionName) bool { _, ok := cn.PeerExtensionIDs[ext] return ok +} + +func (cn *PeerConn) StatusFlags() (ret string) { + return cn.statusFlags() } // Inspired by https://github.com/transmission/transmission/wiki/Peer-Status-Text. diff --git a/storage/file-client.go b/storage/file-client.go index 793b7e58d90e31bd0ad261bee9139ec83999e37e..2a7b5b03e60c47555615e5a35be439c5ed8cd282 100644 --- a/storage/file-client.go +++ b/storage/file-client.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "log/slog" - "os" "path/filepath" g "github.com/anacrolix/generics" @@ -37,7 +36,7 @@ } // The specific part-files option or the default. func (me NewFileClientOpts) partFiles() bool { - return me.UsePartFiles.UnwrapOr(true) + return false } // NewFileOpts creates a new ClientImplCloser that stores files using the OS native filesystem. @@ -72,25 +71,12 @@ return me.opts.PieceCompletion.Close() } var defaultFileIo func() fileIo = func() fileIo { - return &mmapFileIo{} + return &classicFileIo{} } func init() { - s, ok := os.LookupEnv("TORRENT_STORAGE_DEFAULT_FILE_IO") - if !ok { - return - } - switch s { - case "mmap": - defaultFileIo = func() fileIo { - return &mmapFileIo{} - } - case "classic": - defaultFileIo = func() fileIo { - return classicFileIo{} - } - default: - panic(s) + defaultFileIo = func() fileIo { + return classicFileIo{} } } diff --git a/tracker_scraper.go b/tracker_scraper.go index c214f8d596a20005c546578089db0f33cbbf4fc4..81b757ee2b86c7a753f1d8fdd251fbbd12a57fd3 100644 --- a/tracker_scraper.go +++ b/tracker_scraper.go @@ -183,12 +183,6 @@ ClientIp6: krpc.NodeAddr{IP: me.t.cl.config.PublicIp6}, Logger: me.t.logger, }.Do() if err != nil { - level := slog.LevelWarn - if ctx.Err() != nil { - level = slog.LevelDebug - } - // We log here because the caller only stores the error for tracking state. - me.logger.Log(ctx, level, "announce failed", "err", err) ret.Err = fmt.Errorf("announcing: %w", err) return } else {