Skip holes with mmapFileIo

diff --git a/storage/file-io-classic.go b/storage/file-io-classic.go
index bb54e5a773a01c1d00a7e9d5eb8402278a933ce4..5c5ea7a65d8511f54a691e5a88cbb3402904efd1 100644 (file)
--- a/storage/file-io-classic.go
+++ b/storage/file-io-classic.go
@@ -1,6 +1,7 @@
 package storage
 
 import (
+       "io"
        "os"
 )
 
@@ -27,6 +28,10 @@ type classicFileReader struct {
        *os.File
 }
 
-func (c classicFileReader) seekData(offset int64) (ret int64, err error) {
-       return seekData(c.File, offset)
+func (c classicFileReader) seekDataOrEof(offset int64) (ret int64, err error) {
+       ret, err = seekData(c.File, offset)
+       if err == io.EOF {
+               ret, err = c.File.Seek(0, io.SeekEnd)
+       }
+       return
 }

diff --git a/storage/file-io-mmap.go b/storage/file-io-mmap.go
index 422c1c0af118ad026fa26022f2091e4222490ab9..3e1d1277f6d12c71eb915aaab08cf78cdc600d0a 100644 (file)
--- a/storage/file-io-mmap.go
+++ b/storage/file-io-mmap.go
@@ -1,6 +1,7 @@
 package storage
 
 import (
+       "errors"
        "fmt"
        "io"
        "io/fs"
@@ -35,17 +36,22 @@ func (me *mmapFileIo) flush(name string, offset, nbytes int64) error {
 
 type fileMmap struct {
        m        mmap.MMap
-       writable bool
+       f        *os.File
        refs     atomic.Int32
+       writable bool
 }
 
 func (me *fileMmap) dec() error {
        if me.refs.Add(-1) == 0 {
-               return me.m.Unmap()
+               return me.close()
        }
        return nil
 }
 
+func (me *fileMmap) close() (err error) {
+       return errors.Join(me.m.Unmap(), me.f.Close())
+}
+
 func (me *fileMmap) inc() {
        panicif.LessThanOrEqual(me.refs.Add(1), 0)
 }
@@ -75,13 +81,13 @@ func (m *mmapFileIo) openReadOnly(name string) (_ *mmapSharedFileHandle, err err
        if err != nil {
                return
        }
-       defer f.Close()
        mm, err := mmap.Map(f, mmap.RDONLY, 0)
        if err != nil {
+               f.Close()
                err = fmt.Errorf("mapping file: %w", err)
                return
        }
-       v = m.addNewMmap(name, mm, false)
+       v = m.addNewMmap(name, mm, false, f)
        return newMmapFile(v), nil
 }
 
@@ -98,11 +104,17 @@ func (m *mmapFileIo) openForWrite(name string, size int64) (_ fileWriter, err er
                        g.MustDelete(m.paths, name)
                }
        }
+       // TODO: A bunch of this can be done without holding the lock.
        f, err := openFileExtra(name, os.O_RDWR)
        if err != nil {
                return
        }
-       defer f.Close()
+       closeFile := true
+       defer func() {
+               if closeFile {
+                       f.Close()
+               }
+       }()
        err = f.Truncate(size)
        if err != nil {
                err = fmt.Errorf("error truncating file: %w", err)
@@ -118,7 +130,8 @@ func (m *mmapFileIo) openForWrite(name string, size int64) (_ fileWriter, err er
                mm.Unmap()
                return
        }
-       return newMmapFile(m.addNewMmap(name, mm, true)), nil
+       closeFile = false
+       return newMmapFile(m.addNewMmap(name, mm, true, f)), nil
 }
 
 func newMmapFile(f *fileMmap) *mmapSharedFileHandle {
@@ -129,9 +142,10 @@ func newMmapFile(f *fileMmap) *mmapSharedFileHandle {
        return ret
 }
 
-func (me *mmapFileIo) addNewMmap(name string, mm mmap.MMap, writable bool) *fileMmap {
+func (me *mmapFileIo) addNewMmap(name string, mm mmap.MMap, writable bool, f *os.File) *fileMmap {
        v := &fileMmap{
                m:        mm,
+               f:        f,
                writable: writable,
        }
        // One for the store, one for the caller.
@@ -213,8 +227,19 @@ func (me *mmapFileHandle) Read(p []byte) (n int, err error) {
        return
 }
 
-func (me *mmapFileHandle) seekData(offset int64) (ret int64, err error) {
-       me.pos = offset
-       ret = offset
+func (me *mmapFileHandle) seekDataOrEof(offset int64) (ret int64, err error) {
+       // This should be fine as it's an atomic operation, on a shared file handle, so nobody will be
+       // relying non-atomic operations on the file. TODO: Does this require msync first so we don't
+       // skip our own writes.
+       ret, err = seekData(me.shared.f.f, offset)
+       if err == nil {
+               me.pos = ret
+       } else if err == io.EOF {
+               err = nil
+               ret = int64(len(me.shared.f.m))
+               me.pos = ret
+       } else {
+               ret = me.pos
+       }
        return
 }

diff --git a/storage/file-io.go b/storage/file-io.go
index f7c4495eb6dda458f8d663a775e1bbec79ac574b..0f86f0f6879dfd1bd4d3eb4e365db8b3ba3ea1af 100644 (file)
--- a/storage/file-io.go
+++ b/storage/file-io.go
@@ -10,9 +10,8 @@ type fileWriter interface {
 }
 
 type fileReader interface {
-       // Seeks to the next data in the file. If hole-seeking/sparse-files are not supported, should
-       // seek to the offset.
-       seekData(offset int64) (ret int64, err error)
+       // Seeks to the next data in the file. If there is no more data, seeks to the end of the file.
+       seekDataOrEof(offset int64) (ret int64, err error)
        io.WriterTo
        io.ReadCloser
 }

diff --git a/storage/file-piece.go b/storage/file-piece.go
index 76ec477e25c30c3c9241dffbb2c166894fb89bf1..5173bb79d35812d196edd8d17f321963165ed426 100644 (file)
--- a/storage/file-piece.go
+++ b/storage/file-piece.go
@@ -354,12 +354,15 @@ func (me *filePieceImpl) writeFileTo(w io.Writer, fileIndex int, extent segments
        panicif.GreaterThan(extent.End(), file.FileInfo.Length)
        extentRemaining := extent.Length
        var dataOffset int64
-       dataOffset, err = f.seekData(extent.Start)
-       if err == io.EOF {
+       dataOffset, err = f.seekDataOrEof(extent.Start)
+       if err != nil {
+               err = fmt.Errorf("seeking to start of extent: %w", err)
+               return
+       }
+       if dataOffset < extent.Start {
+               // File is too short.
                return
        }
-       panicif.Err(err)
-       panicif.LessThan(dataOffset, extent.Start)
        if dataOffset > extent.Start {
                // Write zeroes until the end of the hole we're in.
                var n1 int64

diff --git a/storage/sys_unix.go b/storage/sys_unix.go
index f294598cabe0248c56a3e10eefbc8d46e5f798c1..31f7d8180bce13e02c8fbe316ba675db1ee688c7 100644 (file)
--- a/storage/sys_unix.go
+++ b/storage/sys_unix.go
@@ -10,6 +10,8 @@ import (
        "golang.org/x/sys/unix"
 )
 
+// Returns io.EOF if there's no data after offset. That doesn't mean there isn't zeroes for a sparse
+// hole. Note that lseek returns -1 on error.
 func seekData(f *os.File, offset int64) (ret int64, err error) {
        ret, err = unix.Seek(int(f.Fd()), offset, unix.SEEK_DATA)
        // TODO: Handle filesystems that don't support sparse files.