Note about possible "out of sync"

[glocate.git] / dirsizer.go

diff --git a/dirsizer.go b/dirsizer.go
index b4f5b4e51c46905addae4a0e5a061a445d23f1a3d6e0e6ce0967c2f36d221a93..ffc8f51ac7c796557f521c4293c4b250ec8d4c4fcbaaea98fe8f3884d375f0d7 100644 (file)
--- a/dirsizer.go
+++ b/dirsizer.go
@@ -1,3 +1,18 @@
+// glocate -- ZFS-diff-friendly locate-like utility
+// Copyright (C) 2022-2024 Sergey Matveev <stargrave@stargrave.org>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, version 3 of the License.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 package main
 
 import (
@@ -6,6 +21,8 @@ import (
        "io"
        "log"
        "os"
+
+       "github.com/klauspost/compress/zstd"
 )
 
 func dirSizer(dirSizes *[]int64, depth int, sinkBack, sinkIn, sinkOut chan Ent) (curSize int64) {
@@ -46,16 +63,27 @@ func applyDirSizes(src *os.File, dirSizes []int64) *os.File {
        if err != nil {
                log.Fatalln(err)
        }
-       tmp, err := os.CreateTemp("", "glocate-idx")
+       tmp, err := os.CreateTemp(TmpDir, "glocate-idx")
+       if err != nil {
+               log.Fatalln(err)
+       }
+
+       compR, err := zstd.NewReader(src)
+       if err != nil {
+               log.Fatalln(err)
+       }
+       br := bufio.NewReaderSize(compR, 1<<17)
+
+       compW, err := zstd.NewWriter(tmp,
+               zstd.WithEncoderLevel(zstd.SpeedBestCompression))
        if err != nil {
                log.Fatalln(err)
        }
+       bw := bufio.NewWriterSize(compW, 1<<17)
 
-       br := bufio.NewReaderSize(src, 1<<17)
        num := make([]byte, 8)
        var nameLen int
        name := make([]byte, 0, 1<<16)
-       bw := bufio.NewWriterSize(tmp, 1<<17)
        var dirIdx int
        for {
                if _, err = io.ReadFull(br, num[:2]); err != nil {
@@ -67,9 +95,7 @@ func applyDirSizes(src *os.File, dirSizes []int64) *os.File {
                mustWrite(bw, num[:2])
                nameLen = int(binary.BigEndian.Uint16(num[:2]))
                name = name[:nameLen]
-               if _, err = io.ReadFull(br, name); err != nil {
-                       log.Fatalln(err)
-               }
+               mustReadFull(br, name)
                mustWrite(bw, name)
                if _, err = io.CopyN(bw, br, 1+8); err != nil {
                        log.Fatalln(err)
@@ -90,5 +116,9 @@ func applyDirSizes(src *os.File, dirSizes []int64) *os.File {
        if err = bw.Flush(); err != nil {
                log.Fatalln(err)
        }
+       if err = compW.Close(); err != nil {
+               log.Fatalln(err)
+       }
+       compR.Close()
        return tmp
 }