Use faster BLAKE3

diff --git a/README b/README
index 56beec94e99a44c33ef68a54f9e3b1461720471b..2479e5bb683ec6f3202de487bbce0f3b9b52ac8a 100644 (file)
--- a/README
+++ b/README
@@ -34,7 +34,7 @@ There are 3 stages this command will do:
   * read first 4 KiB (one disk sector) of each file
   * if that sector differs, then files are not duplicates
   * read each file's contents sequentially with 128 KiB chunks and
   * read first 4 KiB (one disk sector) of each file
   * if that sector differs, then files are not duplicates
   * read each file's contents sequentially with 128 KiB chunks and

-    calculate BLAKE2b-512 digest
+    calculate BLAKE3-256 digest

 
 Action can be the following:
 
 
 Action can be the following:
 

diff --git a/go.mod b/go.mod
index 547702df29d1247f49b868c6e39ebea12cda9e69..f8b0d41317468bb9e9c0416d2eee1f58d9dffc64 100644 (file)
--- a/go.mod
+++ b/go.mod
@@ -1,11 +1,14 @@
 module go.stargrave.org/sgodup
 
 module go.stargrave.org/sgodup
 

-go 1.21
+go 1.24

 
 require (
 
 require (

-       github.com/dustin/go-humanize v1.0.0
+       github.com/dustin/go-humanize v1.0.1

        go.cypherpunks.su/netstring/v3 v3.0.0
        go.cypherpunks.su/netstring/v3 v3.0.0

-       golang.org/x/crypto v0.25.0
+       lukechampine.com/blake3 v1.4.1

 )
 
 )
 

-require golang.org/x/sys v0.22.0 // indirect
+require (
+       github.com/klauspost/cpuid/v2 v2.2.11 // indirect
+       golang.org/x/sys v0.33.0 // indirect
+)

diff --git a/go.sum b/go.sum
index b788fc352804998d09fac931c44680c763d6554c..a78727928c829a47bb323f0819924e2a22e1b06e 100644 (file)
--- a/go.sum
+++ b/go.sum
@@ -1,8 +1,10 @@
-github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
-github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU=
+github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=

 go.cypherpunks.su/netstring/v3 v3.0.0 h1:wwFjxTb/LZM8cQN/UiOPMO5wcuq4xCQWdLAYz74E6kY=
 go.cypherpunks.su/netstring/v3 v3.0.0/go.mod h1:S9pYNVqT6kL2uXbdHz+yxc+A4sAFxBkjSzu+g6KD0QE=
 go.cypherpunks.su/netstring/v3 v3.0.0 h1:wwFjxTb/LZM8cQN/UiOPMO5wcuq4xCQWdLAYz74E6kY=
 go.cypherpunks.su/netstring/v3 v3.0.0/go.mod h1:S9pYNVqT6kL2uXbdHz+yxc+A4sAFxBkjSzu+g6KD0QE=

-golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
-golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
-golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
-golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
+golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+lukechampine.com/blake3 v1.4.1 h1:I3Smz7gso8w4/TunLKec6K2fn+kyKtDxr/xcQEN84Wg=
+lukechampine.com/blake3 v1.4.1/go.mod h1:QFosUxmjB8mnrWFSNwKmvxHpfY72bmD2tQ0kBMM3kwo=

diff --git a/main.go b/main.go
index e49a52078532a3a1e9ae5a9a19218d3928642d3b..04441315e5396d964f08ecef004d474730a7de0b 100644 (file)
--- a/main.go
+++ b/main.go
@@ -33,7 +33,7 @@ import (
 
        "github.com/dustin/go-humanize"
        "go.cypherpunks.su/netstring/v3"
 
        "github.com/dustin/go-humanize"
        "go.cypherpunks.su/netstring/v3"

-       "golang.org/x/crypto/blake2b"
+       "lukechampine.com/blake3"

 )
 
 const (
 )
 
 const (


@@ -300,10 +300,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.`)
        bufOrig := make([]byte, SectorSize)
        seenDup := make(map[string]struct{}, len(queue)/2)
        seenOrig := make(map[string]struct{}, len(queue)/2)
        bufOrig := make([]byte, SectorSize)
        seenDup := make(map[string]struct{}, len(queue)/2)
        seenOrig := make(map[string]struct{}, len(queue)/2)

-       hasher, err := blake2b.New512(nil)
-       if err != nil {
-               panic(err)
-       }
+       hasher := blake3.New(32, nil)

        rdDup := bufio.NewReaderSize(nil, BufSize)
        rdOrig := bufio.NewReaderSize(nil, BufSize)
        var deduped int
        rdDup := bufio.NewReaderSize(nil, BufSize)
        rdOrig := bufio.NewReaderSize(nil, BufSize)
        var deduped int


@@ -349,7 +346,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.`)
                                        readOrig, readDup,
                                )
                        }
                                        readOrig, readDup,
                                )
                        }

-                       if bytes.Compare(bufDup[:readDup], bufOrig[:readOrig]) != 0 {
+                       if !bytes.Equal(bufDup[:readDup], bufOrig[:readOrig]) {

                                if err = fdOrig.Close(); err != nil {
                                        log.Fatal(err)
                                }
                                if err = fdOrig.Close(); err != nil {
                                        log.Fatal(err)
                                }


@@ -389,7 +386,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.`)
                        if err = fdOrig.Close(); err != nil {
                                log.Fatal(err)
                        }
                        if err = fdOrig.Close(); err != nil {
                                log.Fatal(err)
                        }

-                       if bytes.Compare(hashDup, hasher.Sum(nil)) != 0 {
+                       if !bytes.Equal(hashDup, hasher.Sum(nil)) {

                                continue
                        }
                        link(fi.Path, orig.Path)
                                continue
                        }
                        link(fi.Path, orig.Path)


@@ -403,6 +400,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.`)
                        log.Fatal(err)
                }
        }
                        log.Fatal(err)
                }
        }

+       var err error

        if action == ActNS {
                if err = stdoutW.Flush(); err != nil {
                        log.Fatal(err)
        if action == ActNS {
                if err = stdoutW.Flush(); err != nil {
                        log.Fatal(err)

diff --git a/progress.go b/progress.go
index 59d0faf38c6741858bca67862fedd005ef3eb7d7..b5e56adaf2e94fda9d57f340c98d51253085cfd8 100644 (file)
--- a/progress.go
+++ b/progress.go
@@ -53,7 +53,7 @@ func NewProgress(
                humanize.IBytes(uint64(fullSize)),
                suffixFiles,
                suffixSize,
                humanize.IBytes(uint64(fullSize)),
                suffixFiles,
                suffixSize,

-               make(chan struct{}, 0),
+               make(chan struct{}),

        }
        go p.Run(files, size)
        return p
        }
        go p.Run(files, size)
        return p