README | 2 +- diff.go | 10 +++------- dirsizer.go | 25 ++++++++++++++++++++----- go.mod | 2 ++ go.sum | 2 ++ index.go | 7 +------ main.go | 3 +-- reader.go | 23 ++++++++++++++++------- writer.go | 26 ++++++++++++++++++++------ diff --git a/README b/README index 358d5302a98ae9b877e6acf1c950d413e096a065959e58f8d3a631cabf2c1a37..8c52b350ce33c6e2b4f83ae2dd4fc980b342b8d3fa0b79ac2917988101f77477 100644 --- a/README +++ b/README @@ -18,7 +18,7 @@ memory, but that takes 2-3 GiBs of memory, that is huge amount. Moreover it fully loads it to perform any basic searches. So current implementation uses temporary files and heavy use of data streaming. -Its storage format is trivial: +Its storage format is simple: Zstandard-compressed list of records: * 16-bit BE size of the following name * entity (file, directory, symbolic link, etc) name itself. diff --git a/diff.go b/diff.go index 61ba1ac9f83e9f80cb3ac2a7b81e88079e1e9a2d3a8e8afabd34f0485b739d9f..dda4fd9ded52ee8109d30cbf1974527600de590be43819023d0df4ab8fbb91f4 100644 --- a/diff.go +++ b/diff.go @@ -105,7 +105,7 @@ adds := make([][]string, 0, len(addsNames)+len(rens)) mods := make([]*Ent, 0, len(modsNames)+len(rens)) if len(rens) > 0 { sort.Sort(BySrc(rens)) - go reader(bufio.NewReaderSize(db, 1<<17), entsReader) + go reader(db, entsReader) var ent Ent var ok, met bool for { @@ -183,7 +183,7 @@ defer os.Remove(tmp0.Name()) entsReader = make(chan Ent, 1<<10) entsDirSizer := make(chan Ent, 1<<10) entsWriter := make(chan Ent, 1<<10) - go reader(bufio.NewReaderSize(db, 1<<17), entsReader) + go reader(db, entsReader) dirSizerJob := make(chan struct{}) var dirSizes []int64 @@ -193,10 +193,9 @@ dirSizer(&dirSizes, 1, sinkBack, entsDirSizer, entsWriter) close(dirSizerJob) }() - bw := bufio.NewWriterSize(tmp0, 1<<17) writerJob := make(chan struct{}) go func() { - writer(bw, entsWriter) + writer(tmp0, entsWriter) close(writerJob) }() @@ -243,9 +242,6 @@ close(entsDirSizer) <-dirSizerJob close(entsWriter) <-writerJob - if err = bw.Flush(); err != nil { - log.Fatalln(err) - } tmp1 := applyDirSizes(tmp0, dirSizes) tmp0.Close() diff --git a/dirsizer.go b/dirsizer.go index b4f5b4e51c46905addae4a0e5a061a445d23f1a3d6e0e6ce0967c2f36d221a93..fda37f4fe46098cf687690e5e03984aba77f952ed5814359e94deefcccbbbdb3 100644 --- a/dirsizer.go +++ b/dirsizer.go @@ -6,6 +6,8 @@ "encoding/binary" "io" "log" "os" + + "github.com/klauspost/compress/zstd" ) func dirSizer(dirSizes *[]int64, depth int, sinkBack, sinkIn, sinkOut chan Ent) (curSize int64) { @@ -51,11 +53,22 @@ if err != nil { log.Fatalln(err) } - br := bufio.NewReaderSize(src, 1<<17) + compR, err := zstd.NewReader(src) + if err != nil { + log.Fatalln(err) + } + br := bufio.NewReaderSize(compR, 1<<17) + + compW, err := zstd.NewWriter(tmp, + zstd.WithEncoderLevel(zstd.SpeedBestCompression)) + if err != nil { + log.Fatalln(err) + } + bw := bufio.NewWriterSize(compW, 1<<17) + num := make([]byte, 8) var nameLen int name := make([]byte, 0, 1<<16) - bw := bufio.NewWriterSize(tmp, 1<<17) var dirIdx int for { if _, err = io.ReadFull(br, num[:2]); err != nil { @@ -67,9 +80,7 @@ } mustWrite(bw, num[:2]) nameLen = int(binary.BigEndian.Uint16(num[:2])) name = name[:nameLen] - if _, err = io.ReadFull(br, name); err != nil { - log.Fatalln(err) - } + mustReadFull(br, name) mustWrite(bw, name) if _, err = io.CopyN(bw, br, 1+8); err != nil { log.Fatalln(err) @@ -90,5 +101,9 @@ } if err = bw.Flush(); err != nil { log.Fatalln(err) } + if err = compW.Close(); err != nil { + log.Fatalln(err) + } + compR.Close() return tmp } diff --git a/go.mod b/go.mod index f8654fc71f5b20da772216ab5c99b8f3d5c0d4e6f80285b811eb9ef3a932a85b..7d95a307792df055072c49908b254f1d7104f6872fe699f79df78e2f5f315f2c 100644 --- a/go.mod +++ b/go.mod @@ -3,3 +3,5 @@ go 1.18 require github.com/dustin/go-humanize v1.0.0 + +require github.com/klauspost/compress v1.15.9 diff --git a/go.sum b/go.sum index 4f89ea40df43dd72a9ee4bb17b5304a77cd360409cd05cc9431c6210cfbb63c5..568acaac67a8ceacd643b3927d665ea337dccc412d49088ad5449810553ca241 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,4 @@ github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= +github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= diff --git a/index.go b/index.go index c6fa5398b74fb4ace31a4f09587b8579e3b9642d55040d46e92a1f37fe5e99fb..c05e4c0b84d35c2d6d49beede0faf14c8f985fd98f2d42c1bf11a336f2f49381 100644 --- a/index.go +++ b/index.go @@ -1,7 +1,6 @@ package main import ( - "bufio" "fmt" "log" "os" @@ -25,10 +24,9 @@ dirSizer(&dirSizes, 1, sinkBack, entsWalker, entsWriter) close(dirSizerJob) }() - bw := bufio.NewWriterSize(tmp0, 1<<17) writerJob := make(chan struct{}) go func() { - writer(bw, entsWriter) + writer(tmp0, entsWriter) close(writerJob) }() @@ -45,9 +43,6 @@ close(entsWalker) <-dirSizerJob close(entsWriter) <-writerJob - if err = bw.Flush(); err != nil { - log.Fatalln(err) - } tmp1 := applyDirSizes(tmp0, dirSizes) tmp0.Close() diff --git a/main.go b/main.go index a5fcb41451e9410c3be3d629ba89dea6d354c675abe9fd2fa7e0879514b10095..c8615fdd85d4f2b76db67f755348feb0a9baf32c17bfdab0b72473fdb634dd58 100644 --- a/main.go +++ b/main.go @@ -1,7 +1,6 @@ package main import ( - "bufio" "flag" "log" "os" @@ -64,7 +63,7 @@ if err != nil { log.Fatalln(err) } entsReader := make(chan Ent, 1<<10) - go reader(bufio.NewReaderSize(db, 1<<17), entsReader) + go reader(db, entsReader) entsPrinter := make(chan Ent, 1<<10) printerJob := make(chan struct{}) diff --git a/reader.go b/reader.go index 269fc3295dac975ee17b8a4007c9995da5813562f989a1ff4ce3f92366daa7eb..8900c26a75b3fd91ba83e1d45f9d923436981f3e84a45ea784cb151c474d50b4 100644 --- a/reader.go +++ b/reader.go @@ -1,9 +1,12 @@ package main import ( + "bufio" "encoding/binary" "io" "log" + + "github.com/klauspost/compress/zstd" ) func mustReadFull(r io.Reader, buf []byte) { @@ -12,15 +15,20 @@ log.Fatalln(err) } } -func reader(r io.Reader, sink chan Ent) { - var err error +func reader(src io.Reader, sink chan Ent) { + comp, err := zstd.NewReader(src) + if err != nil { + log.Fatalln(err) + } + br := bufio.NewReaderSize(comp, 1<<17) + num := make([]byte, 8) var cols []string var namePrev string var nameLen uint16 var depth, depthPrev uint8 for { - _, err = io.ReadFull(r, num[:2]) + _, err = io.ReadFull(br, num[:2]) if err != nil { if err == io.EOF { break @@ -29,13 +37,13 @@ log.Fatalln(err) } nameLen = binary.BigEndian.Uint16(num[:2]) nameRaw := make([]byte, nameLen) - mustReadFull(r, nameRaw) + mustReadFull(br, nameRaw) name := string(nameRaw) - mustReadFull(r, num[:1]) + mustReadFull(br, num[:1]) depth = uint8(num[0]) - mustReadFull(r, num) + mustReadFull(br, num) ent := Ent{mtime: int64(binary.BigEndian.Uint64(num))} - mustReadFull(r, num) + mustReadFull(br, num) ent.size = int64(binary.BigEndian.Uint64(num)) if depth > depthPrev { cols = append(cols, namePrev[:len(namePrev)-1]) @@ -48,4 +56,5 @@ namePrev = name depthPrev = depth } close(sink) + comp.Close() } diff --git a/writer.go b/writer.go index e12a19c0d7f2103ae9c88e4c4f3c237bb3d03f62819ea98eb33c93210f221256..75b82e57abc5e88fff375e6c1a4a69ae1335b4e2ce944ad401c940ee08e3c8e1 100644 --- a/writer.go +++ b/writer.go @@ -1,9 +1,12 @@ package main import ( + "bufio" "encoding/binary" "io" "log" + + "github.com/klauspost/compress/zstd" ) func mustWrite(w io.Writer, buf []byte) { @@ -12,7 +15,12 @@ log.Fatalln(err) } } -func writer(w io.Writer, sink chan Ent) { +func writer(dst io.Writer, sink chan Ent) { + comp, err := zstd.NewWriter(dst) + if err != nil { + log.Fatalln(err) + } + bw := bufio.NewWriterSize(comp, 1<<17) num := make([]byte, 8) var name string for ent := range sink { @@ -21,15 +29,21 @@ if len(ent.name) >= 1<<16 { panic("too long") } binary.BigEndian.PutUint16(num[:2], uint16(len(name))) - mustWrite(w, num[:2]) - mustWrite(w, []byte(name)) + mustWrite(bw, num[:2]) + mustWrite(bw, []byte(name)) if len(ent.name) >= 1<<8 { panic("too deep") } - mustWrite(w, []byte{byte(len(ent.name) - 1)}) + mustWrite(bw, []byte{byte(len(ent.name) - 1)}) binary.BigEndian.PutUint64(num, uint64(ent.mtime)) - mustWrite(w, num) + mustWrite(bw, num) binary.BigEndian.PutUint64(num, uint64(ent.size)) - mustWrite(w, num) + mustWrite(bw, num) + } + if err = bw.Flush(); err != nil { + log.Fatalln(err) + } + if err = comp.Close(); err != nil { + log.Fatalln(err) } }