Add mmap alternative IO system for file storage

diff --git a/storage/file-client.go b/storage/file-client.go
index ee3b5efc825976875650a85e4b672f18b54e5bce..3bcbd634d02a888def6275a735ed2d5a533df3b8 100644 (file)
--- a/storage/file-client.go
+++ b/storage/file-client.go
@@ -4,6 +4,7 @@ import (
        "context"
        "fmt"
        "log/slog"
+       "os"
        "path/filepath"
 
        g "github.com/anacrolix/generics"
@@ -70,6 +71,29 @@ func (me *fileClientImpl) Close() error {
        return me.opts.PieceCompletion.Close()
 }
 
+var defaultFileIo func() fileIo = func() fileIo {
+       return classicFileIo{}
+}
+
+func init() {
+       s, ok := os.LookupEnv("TORRENT_STORAGE_DEFAULT_FILE_IO")
+       if !ok {
+               return
+       }
+       switch s {
+       case "mmap":
+               defaultFileIo = func() fileIo {
+                       return &mmapFileIo{}
+               }
+       case "classic":
+               defaultFileIo = func() fileIo {
+                       return classicFileIo{}
+               }
+       default:
+               panic(s)
+       }
+}
+
 func (fs *fileClientImpl) OpenTorrent(
        ctx context.Context,
        info *metainfo.Info,
@@ -104,7 +128,7 @@ func (fs *fileClientImpl) OpenTorrent(
                metainfoFileInfos,
                info.FileSegmentsIndex(),
                infoHash,
-               classicFileIo{},
+               defaultFileIo(),
                fs,
        }
        if t.partFiles() {

diff --git a/storage/file-io.go b/storage/file-io.go
index 5dd6e3c790e3baa2b15da80228c3f5750b869316..a8962e19786b3f9990e3c139e1e5e5e7fcbefc12 100644 (file)
--- a/storage/file-io.go
+++ b/storage/file-io.go
@@ -1,11 +1,7 @@
 package storage
 
 import (
-       "errors"
        "io"
-       "io/fs"
-       "os"
-       "path/filepath"
 )
 
 type fileWriter interface {
@@ -24,46 +20,5 @@ type fileReader interface {
 type fileIo interface {
        openForSharedRead(name string) (sharedFileIf, error)
        openForRead(name string) (fileReader, error)
-       openForWrite(name string) (fileWriter, error)
-}
-
-type classicFileIo struct{}
-
-func (i classicFileIo) openForSharedRead(name string) (sharedFileIf, error) {
-       return sharedFiles.Open(name)
-}
-
-type classicFileReader struct {
-       *os.File
-}
-
-func (c classicFileReader) seekData(offset int64) (ret int64, err error) {
-       return seekData(c.File, offset)
-}
-
-func (i classicFileIo) openForRead(name string) (fileReader, error) {
-       f, err := os.Open(name)
-       return classicFileReader{f}, err
-}
-
-func (classicFileIo) openForWrite(p string) (f fileWriter, err error) {
-       f, err = os.OpenFile(p, os.O_WRONLY|os.O_CREATE, filePerm)
-       if err == nil {
-               return
-       }
-       if errors.Is(err, fs.ErrNotExist) {
-               err = os.MkdirAll(filepath.Dir(p), dirPerm)
-               if err != nil {
-                       return
-               }
-       } else if errors.Is(err, fs.ErrPermission) {
-               err = os.Chmod(p, filePerm)
-               if err != nil {
-                       return
-               }
-       } else {
-               return
-       }
-       f, err = os.OpenFile(p, os.O_WRONLY|os.O_CREATE, filePerm)
-       return
+       openForWrite(name string, size int64) (fileWriter, error)
 }

diff --git a/storage/file-piece.go b/storage/file-piece.go
index ee2da24e7dd2850bc08d60ee79da89eca86466f8..53f4af86c57ce84e5ddc8e94df5b4333e24bc8a1 100644 (file)
--- a/storage/file-piece.go
+++ b/storage/file-piece.go
@@ -299,6 +299,24 @@ var (
        packageExpvarMap = expvar.NewMap("torrentStorage")
 )
 
+type limitWriter struct {
+       rem int64
+       w   io.Writer
+}
+
+func (me *limitWriter) Write(p []byte) (n int, err error) {
+       n, err = me.w.Write(p[:min(int64(len(p)), me.rem)])
+       me.rem -= int64(n)
+       if err != nil {
+               return
+       }
+       p = p[n:]
+       if len(p) > 0 {
+               err = io.ErrShortWrite
+       }
+       return
+}
+
 func (me *filePieceImpl) writeFileTo(w io.Writer, fileIndex int, extent segments.Extent) (written int64, err error) {
        if extent.Length == 0 {
                return
@@ -337,7 +355,18 @@ func (me *filePieceImpl) writeFileTo(w io.Writer, fileIndex int, extent segments
                extentRemaining -= n1
        }
        var n1 int64
-       n1, err = io.CopyN(w, f, extentRemaining)
+       if true {
+               n1, err = f.WriteTo(&limitWriter{
+                       rem: extentRemaining,
+                       w:   w,
+               })
+               // limitWriter will block f from writing too much.
+               if n1 == extentRemaining {
+                       err = nil
+               }
+       } else {
+               n1, err = io.CopyN(w, f, extentRemaining)
+       }
        packageExpvarMap.Add("bytesReadNotSkipped", n1)
        written += n1
        return

diff --git a/storage/file-torrent.go b/storage/file-torrent.go
index 0c753ed2f70ba5b6b7eb2e812b2f493cd9990127..54237a70adc97a2c853426a84907b52fe84e9423 100644 (file)
--- a/storage/file-torrent.go
+++ b/storage/file-torrent.go
@@ -164,8 +164,8 @@ func (me *fileTorrentImpl) openFile(file file) (f fileReader, err error) {
        return
 }
 
-func (fst fileTorrentImpl) openForWrite(file file) (_ fileWriter, err error) {
+func (fst *fileTorrentImpl) openForWrite(file file) (_ fileWriter, err error) {
        // It might be possible to have a writable handle shared files cache if we need it.
        fst.logger().Debug("openForWrite", "file.safeOsPath", file.safeOsPath)
-       return fst.io.openForWrite(fst.pathForWrite(&file))
+       return fst.io.openForWrite(fst.pathForWrite(&file), file.FileInfo.Length)
 }

diff --git a/zero-reader.go b/zero-reader.go
index 1d0a899b3afa1b01551edeb24d232b1108e2e1cc..92fc8059b253dcf0843701c15ddcae51aca2ddab 100644 (file)
--- a/zero-reader.go
+++ b/zero-reader.go
@@ -1,5 +1,6 @@
 package torrent
 
+// TODO: This should implement extra methods to make io.CopyN more efficient.
 var zeroReader zeroReaderType
 
 type zeroReaderType struct{}