From be69a4fc0c240c2772a0f0bf0955b39783e4c48a Mon Sep 17 00:00:00 2001 From: Sergey Matveev Date: Thu, 7 Mar 2024 16:57:08 +0300 Subject: [PATCH] *-hashes-detect, *-hash, *-check -stdin/-all-hashes, optional builtins --- HASHES | 62 +++++++++ INSTALL | 3 + README | 36 +---- bin/meta4ra-check | 1 + bin/meta4ra-create | 1 + bin/meta4ra-hash | 1 + bin/meta4ra-hashes-detect | 67 +++++++++ build | 3 + build-with-thirdparty | 3 + cmd/{meta4-check/main.go => meta4ra/check.go} | 123 ++++++++++++----- .../main.go => meta4ra/create.go} | 27 ++-- cmd/meta4ra/hash.go | 63 +++++++++ cmd/meta4ra/main.go | 21 +++ contrib/mk-meta4 | 12 ++ contrib/shake128sum | 4 - contrib/shake256sum | 4 - contrib/streebog256sum | 3 - contrib/streebog512sum | 3 - go.mod | 13 ++ go.sum | 15 +++ hasher.go | 108 --------------- internal/builtin.go | 59 ++++++++ common.go => internal/common.go | 5 +- internal/hasher.go | 127 ++++++++++++++++++ scheme.go => internal/scheme.go | 2 +- internal/thirdparty.go | 95 +++++++++++++ internal/xxh3.go | 26 ++++ 27 files changed, 689 insertions(+), 198 deletions(-) create mode 100644 HASHES create mode 100644 INSTALL create mode 120000 bin/meta4ra-check create mode 120000 bin/meta4ra-create create mode 120000 bin/meta4ra-hash create mode 100755 bin/meta4ra-hashes-detect create mode 100755 build create mode 100755 build-with-thirdparty rename cmd/{meta4-check/main.go => meta4ra/check.go} (59%) rename cmd/{meta4-create/main.go => meta4ra/create.go} (88%) create mode 100644 cmd/meta4ra/hash.go create mode 100644 cmd/meta4ra/main.go create mode 100755 contrib/mk-meta4 delete mode 100755 contrib/shake128sum delete mode 100755 contrib/shake256sum delete mode 100755 contrib/streebog256sum delete mode 100755 contrib/streebog512sum create mode 100644 go.sum delete mode 100644 hasher.go create mode 100644 internal/builtin.go rename common.go => internal/common.go (91%) create mode 100644 internal/hasher.go rename scheme.go => internal/scheme.go (99%) create mode 100644 internal/thirdparty.go create mode 100644 internal/xxh3.go diff --git a/HASHES b/HASHES new file mode 100644 index 0000000..3198147 --- /dev/null +++ b/HASHES @@ -0,0 +1,62 @@ +There are plenty of possible hashing algorithms in use. Some are forced +by standards, some are faster on hardware, some on 64-bit CPUs, some are +relatively new. So there is no single algorithm that will satisfy you in +every situation. That is why, Metalink4 files can contain many choices +of the hashes. + +Various utilities, various versions, various OS distributions have +different set of available/better options. That is why meta4ra-create +and meta4ra-check utilities have -hashes option, where you specify set +of supported algorithms and command line to run for their calculation. +-hashes is comma-separated list of colon-separated "name:cmdline" pairs. + -hashes "skein-512:skein512,sha512:libressl dgst -sha512" +option tells, that for calculation of Skein-512 you have to run skein512 +command, and for SHA2-512 "libressl ..." one. They are invoked under +"/bin/sh -e -c" command, so pipelines are also allowable there. Data is +fed to their stdout and they are expected to print hash value in +hexadecimal form as a first (or single) column to stdout. First found +common algorithm is used by default for file verification in +meta4ra-check utility, so order in -hashes is important. + +meta4ra-hashes-detect utility conveniently checks various predefined +known commands and outputs -hashes-compatible string for your system. + +If you use "builtin" word as a command, then builtin implementation of +the hash will be used. By default, meta4ra does not require any +non-standard library dependencies, so it includes only SHA2-256 and +SHA2-512. Optionally you can run build-with-thirdparty script to enable +building with third-party libraries, including much more other hashes. + +Only a few hashes are standardised: +https://www.iana.org/assignments/hash-function-text-names/hash-function-text-names.xhtml +But meta4ra uses more of advanced and performant ones. They are listed +below in order of preference. + +* blake3-256 + Ultimately fast hash, still considered cryptographically secure. + Out-of-box Merkle-tree gives ability to indefinitely parallelise for + >1KiB blocks. It runs faster than either MD5 or hardware-accelerated + SHA2-256. Runs several times faster with AVX512. It is reduced round + BLAKE2s. +* blake2b-512, blake2b-256 + Very fast hash, pretty widespread as a SHA2 replacement. + Both of its 512/256-bit variants are often met. + It is reduced round BLAKE, that was among SHA3 finalists. +* skein-512 + Skein is fastest software hash among all SHA3 finalists, with huge + security margin. +* shake128, shake256 + SHAKE is the officially recommended SHA3 mode of operation for general + usage. SHAKE256 as fast as software SHA2-512 with the comparable + security level. Can be very fast on specialised hardware. +* sha-512, sha-256 + SHA2 is rather slow, but has widespread availability. 512-bit version + version runs faster on 64-bit CPUs. However modern CPUs have hardware + accelerated SHA2-256, making it slower only than BLAKE3 +* streebog-512, streebog-256 + Russian Federation's government standard for hashing. Both versions + have identical speed, so 512-bit is preferred. +* xxh3-128 + XXH3 is not a cryptographically secure hash (that is why it is at the + very end of the list), but 128-bit output with the speed of RAM makes + it also useful for integrity checking diff --git a/INSTALL b/INSTALL new file mode 100644 index 0000000..48bf1fb --- /dev/null +++ b/INSTALL @@ -0,0 +1,3 @@ +Run "build" script to put compiled binary to bin/ directory. +"build-with-thirdparty" builds them using third-party libraries +for more builtin hashes choices. diff --git a/README b/README index 1969527..15bc1a4 100644 --- a/README +++ b/README @@ -1,39 +1,15 @@ meta4ra -- Metalink4 utilities -meta4-create utility is used to create Metalink4 +meta4ra-create utility is used to create Metalink4 (https://datatracker.ietf.org/doc/html/rfc5854) .meta4-file for single specified file. -Look for its -help for invocation information. --hashes are comma-separated list of colon-separated pairs of hash'es -name and the external command used to compute it. Command shall take -the data from stdin and print the digest in hexadecimal form with the -following newline to stdout. +meta4ra-check utility is used to check .meta4 file, extract signatures +and verify corresponding files integrity. -Following hashes are predefined by default: - -* sha-256:sha256 -- standardized name; standard command in many OSes - May be fast on hardware accelerated CPUs. -* sha-512:sha512 -- standardized name; standard command in many OSes - Faster on 64-bit CPUs than software sha-256. -* skein-256:skein256 -- non-standardized name; out-of-box command in FreeBSD - Faster than software sha-*/shake*. -* skein-512:skein512 -- non-standardized name; out-of-box command in FreeBSD - Faster on 64-bit CPUs than skein-256. -* shake128:goshake128 -- standardized name; non-standard command - Faster than software sha-*. Much faster on hardware. -* shake256:goshake256 -- standardized name; non-standard command - Same speed as sha-512 on 64-bit CPUs. - Much faster on hardware. -* streebog-256:streebog256sum -- non-standardized name; command is in contrib/ -* streebog-512:streebog512sum -- non-standardized name; command is in contrib/ -* blake3-256:b3sum -- non-standardized name; additional package in most OSes - Fastest parallelizeable software hash - -SHA2 and SHA3 (SHAKE*) are USA's NIST standards. Streebog is Russian -Federation's government standard. Skein is SHA3-finalist. BLAKE3 is -reduced round Merklee-tree-based BLAKE2 descendant. BLAKE2 is reduced -round BLAKE, that also was one of SHA3-finalists. +meta4ra-hash utility can be used to hash the data with a single hash +algorithm. It could be useful if you want to use the best found +algorithm on current system, whatever it is. meta4ra is copylefted free software: see the file COPYING for copying conditions. It should work on all POSIX-compatible systems. diff --git a/bin/meta4ra-check b/bin/meta4ra-check new file mode 120000 index 0000000..ebcf0ba --- /dev/null +++ b/bin/meta4ra-check @@ -0,0 +1 @@ +meta4ra \ No newline at end of file diff --git a/bin/meta4ra-create b/bin/meta4ra-create new file mode 120000 index 0000000..ebcf0ba --- /dev/null +++ b/bin/meta4ra-create @@ -0,0 +1 @@ +meta4ra \ No newline at end of file diff --git a/bin/meta4ra-hash b/bin/meta4ra-hash new file mode 120000 index 0000000..ebcf0ba --- /dev/null +++ b/bin/meta4ra-hash @@ -0,0 +1 @@ +meta4ra \ No newline at end of file diff --git a/bin/meta4ra-hashes-detect b/bin/meta4ra-hashes-detect new file mode 100755 index 0000000..20d4815 --- /dev/null +++ b/bin/meta4ra-hashes-detect @@ -0,0 +1,67 @@ +#!/bin/sh -e + +hw="hello world" +hashes="" + +desired="$1" + +check() { + local name="$1" + if [ -n "$desired" ] && [ $desired != "$name" ] ; then return 1 ; fi + local cmd="$2" + local hsh="$3" + our=$(echo -n hello world | sh -e -c "$cmd" | { read h rem ; printf %s "$h"; }) + [ $hsh = "$our" ] && hashes="$hashes,$name:$cmd" +} + +hsh=d74981efa70a0c880b8d8c1985d075dbcbf679b99a5f9914e5aaf96b831a9e24 +check blake3-256 b3sum $hsh || : + +hsh=021ced8799296ceca557832ab941a50b4a11f83478cf141f51f933f653ab9fbcc05a037cddbed06e309bf334942c4e58cdf1a46e237911ccd7fcf9787cbc7fd0 +check blake2b-512 b2sum $hsh || +check blake2b-512 "openssl blake2b512 -r" $hsh || : + +hsh=256c83b297114d201b30179f3f0ef0cace9783622da5974326b436178aeef610 +check blake2b-256 "b2sum -l 256" $hsh || : + +hsh=8b4830244fc36daa11177311dc6bf7636376180dce2d29193335878142e7d6f5e9016beba729e0a353dd2fd421c8b2022ee8927f0bce6b88631bb01be2e0f5ba +check skein-512 skein512 $hsh || : + +hsh=3a9159f071e4dd1c8c4f968607c30942e120d8156b8b1e72e0d376e8871cb8b8 +check shake128 goshake128 $hsh || # go.stargrave.org/gosha3 +check shake128 "sha3sum -a 128000 | dd bs=1 count=64 2> /dev/null" $hsh || : # p5-Digest-SHA3 +# openssl shake128 -- useless, as it outputs only 128 bits + +hsh=369771bb2cb9d2b04c1d54cca487e372d9f187f73f7ba3f65b95c8ee7798c527f4f3c2d55c2d46a29f2e945d469c3df27853a8735271f5cc2d9e889544357116 +check shake256 goshake256 $hsh || +check shake256 "sha3sum -a 256000 | dd bs=1 count=128 2> /dev/null" $hsh || : + +hsh=309ecc489c12d6eb4cc40f50c902f2b4d0ed77ee511a7c7a9bcd3ca86d4cd86f989dd35bc5ff499670da34255b45b0cfd830e81f605dcf7dc5542e93ae9cd76f +check sha-512 sha512 $hsh || +check sha-512 sha512sum $hsh || +check sha-512 "libressl dgst -sha512" $hsh || +check sha-512 "openssl sha512 -r" $hsh || : + +hsh=b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9 +check sha-256 sha256 $hsh || +check sha-256 sha256sum $hsh || +check sha-256 "libressl dgst -sha256" $hsh || +check sha-256 "openssl sha256 -r" $hsh || : + +hsh=84d883ede9fa6ce855d82d8c278ecd9f5fc88bf0602831ae0c38b9b506ea3cb02f3fa076b8f5664adf1ff862c0157da4cc9a83e141b738ff9268a9ba3ed6f563 +check streebog-512 "nettle-hash --algorithm=streebog512 --raw | xxd -c 0 -p" $hsh || +check streebog-512 "nettle-hash --algorithm=streebog512 --raw | hexdump -v -e '/1 \"%02x\"' ; echo" $hsh || +check streebog-512 "libressl dgst -streebog512" $hsh || +check streebog-512 streebog512 $hsh || : # go.cypherpunks.ru/gogost + +hsh=c600fd9dd049cf8abd2f5b32e840d2cb0e41ea44de1c155dcd88dc84fe58a855 +check streebog-256 "nettle-hash --algorithm=streebog256 --raw | xxd -c 0 -p" $hsh || +check streebog-256 "nettle-hash --algorithm=streebog256 --raw | hexdump -v -e '/1 \"%02x\"' ; echo" $hsh || +check streebog-256 "libressl dgst -streebog256" $hsh || +check streebog-256 streebog256 $hsh || : + +hsh=df8d09e93f874900a99b8775cc15b6c7 +check xxh3-128 "xxhsum -H128" $hsh || : + +[ -n "$hashes" ] +echo ${hashes#,} diff --git a/build b/build new file mode 100755 index 0000000..32381dd --- /dev/null +++ b/build @@ -0,0 +1,3 @@ +#!/bin/sh -e + +exec go build -C cmd/meta4ra -ldflags=-s $@ -o ../../bin/meta4ra diff --git a/build-with-thirdparty b/build-with-thirdparty new file mode 100755 index 0000000..e24796f --- /dev/null +++ b/build-with-thirdparty @@ -0,0 +1,3 @@ +#!/bin/sh -e + +exec ./build -tags thirdparty diff --git a/cmd/meta4-check/main.go b/cmd/meta4ra/check.go similarity index 59% rename from cmd/meta4-check/main.go rename to cmd/meta4ra/check.go index b807a26..c030f8b 100644 --- a/cmd/meta4-check/main.go +++ b/cmd/meta4ra/check.go @@ -13,7 +13,6 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -// Metalink 4.0 checker package main import ( @@ -28,12 +27,14 @@ import ( "path" "strings" - "go.stargrave.org/meta4ra" + meta4ra "go.stargrave.org/meta4ra/internal" ) -func main() { - hashes := flag.String("hashes", - strings.Join(meta4ra.HashesDefault, ","), "hash-name:command-s") +func runCheck() { + stdin := flag.Bool("stdin", false, "Compare data of single file taken from stdin") + allHashes := flag.Bool("all-hashes", false, "Check all hashes, not the first common one") + hashes := flag.String("hashes", meta4ra.HashesDefault, + "hash-name:commandline[,...]") extractSig := flag.Bool("extract-sig", false, "Extract signature files") metaPath := flag.String("meta4", "file.meta4", "Metalink file") flag.Usage = func() { @@ -44,11 +45,10 @@ func main() { If no FILEs are specified, then all s from metalink are searched and verified if they exist. Otherwise only specified FILEs are checked. If you want to skip any verification (for example only to validate the format -and -extract-sig, then you can just specify empty ("") FILE. +and -extract-sig, then you can just specify an empty ("") FILE. `) } flag.Parse() - log.SetFlags(log.Lshortfile) data, err := os.ReadFile(*metaPath) if err != nil { @@ -64,6 +64,9 @@ and -extract-sig, then you can just specify empty ("") FILE. for _, fn := range flag.Args() { toCheck[path.Base(fn)] = fn } + if *stdin && len(toCheck) != 1 { + log.Fatalln("exactly single FILE must be specified when using -stdin") + } bad := false for _, f := range meta.Files { @@ -92,28 +95,39 @@ and -extract-sig, then you can just specify empty ("") FILE. } fullPath := toCheck[f.Name] + delete(toCheck, f.Name) if !(len(toCheck) == 0 || fullPath != "") { continue } if fullPath == "" { fullPath = f.Name } - s, err := os.Stat(fullPath) - if err != nil { - fmt.Println(err) - bad = true - continue + if !*stdin { + s, err := os.Stat(fullPath) + if err != nil { + fmt.Println(err) + bad = true + continue + } + if uint64(s.Size()) != f.Size { + fmt.Println("size mismatch", + f.Name, "our:", s.Size(), "their:", f.Size) + bad = true + continue + } } - if uint64(s.Size()) != f.Size { - fmt.Println("size mismatch", - f.Name, "our:", s.Size(), "their:", f.Size) + + hasher, err := meta4ra.NewHasher(*hashes) + if err != nil { + fmt.Println(f.Name, err) bad = true continue } - - hasher := meta4ra.NewHasher(*hashes) var hashTheir string var hashName string + if *allHashes { + goto HashFound + } for i, name := range hasher.Names { for _, h := range f.Hashes { if h.Type == name { @@ -130,35 +144,80 @@ and -extract-sig, then you can just specify empty ("") FILE. fmt.Println("no common hashes found for:", f.Name) bad = true continue - HashFound: - fd, err := os.Open(fullPath) + + fd := os.Stdin + if !*stdin { + fd, err = os.Open(fullPath) + if err != nil { + fmt.Println("Error:", f.Name, err) + bad = true + continue + } + } + err = hasher.Start() if err != nil { + if !*stdin { + fd.Close() + } + hasher.Stop() fmt.Println("Error:", f.Name, err) bad = true continue } - hasher.Start() - _, err = io.Copy(hasher, bufio.NewReaderSize(fd, 1<<20)) - fd.Close() - sums := hasher.Sums() + _, err = io.Copy(hasher, bufio.NewReaderSize(fd, meta4ra.BufLen)) + if !*stdin { + fd.Close() + } if err != nil { + hasher.Stop() fmt.Println("Error:", f.Name, err) bad = true continue } - hashOur := sums[0].Hash - if hashOur == hashTheir { - fmt.Println(f.Name, hashName, "good") - } else { - fmt.Println( - "hash mismatch:", f.Name, hashName, - "our:", hashOur, - "their:", hashTheir, - ) + sums, err := hasher.Sums() + if err != nil { + hasher.Stop() + fmt.Println("Error:", f.Name, err) bad = true continue } + if *allHashes { + hashesOur := make(map[string]string, len(sums)) + for _, h := range sums { + hashesOur[h.Type] = h.Hash + } + for _, h := range f.Hashes { + hashOur := hashesOur[h.Type] + if h.Hash == hashOur { + fmt.Println(f.Name, h.Type, "good") + } else { + fmt.Println( + "hash mismatch:", f.Name, h.Type, + "our:", hashOur, + "their:", h.Hash, + ) + bad = true + } + } + } else { + hashOur := sums[0].Hash + if hashOur == hashTheir { + fmt.Println(f.Name, hashName, "good") + } else { + fmt.Println( + "hash mismatch:", f.Name, hashName, + "our:", hashOur, + "their:", hashTheir, + ) + bad = true + continue + } + } + } + if len(toCheck) != 0 { + fmt.Println("not all FILEs met") + bad = true } if bad { os.Exit(1) diff --git a/cmd/meta4-create/main.go b/cmd/meta4ra/create.go similarity index 88% rename from cmd/meta4-create/main.go rename to cmd/meta4ra/create.go index 1e8b69e..cfa7f4e 100644 --- a/cmd/meta4-create/main.go +++ b/cmd/meta4ra/create.go @@ -13,7 +13,6 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -// Metalink 4.0 creator package main import ( @@ -25,13 +24,12 @@ import ( "log" "os" "path" - "strings" "time" - "go.stargrave.org/meta4ra" + meta4ra "go.stargrave.org/meta4ra/internal" ) -func main() { +func runCreate() { fn := flag.String("fn", "", "Filename") mtime := flag.String("mtime", "", "Take that file's mtime as a Published date") desc := flag.String("desc", "", "Description") @@ -39,8 +37,8 @@ func main() { "Path to OpenPGP .asc signature file for inclusion") sigSSH := flag.String("sig-ssh", "", "Path to OpenSSH .sig signature file for inclusion") - hashes := flag.String("hashes", - strings.Join(meta4ra.HashesDefault, ","), "hash-name:command-s") + hashes := flag.String("hashes", meta4ra.HashesDefault, + "hash-name:commandline[,...]") noPublished := flag.Bool("no-published", false, "Do not include Published field") noGenerator := flag.Bool("no-generator", false, @@ -52,7 +50,7 @@ func main() { flag.PrintDefaults() } flag.Parse() - log.SetFlags(log.Lshortfile) + if *fn == "" { log.Fatalln("empty -fn") } @@ -60,20 +58,27 @@ func main() { for _, u := range flag.Args() { urls = append(urls, meta4ra.URL{URL: u}) } - h := meta4ra.NewHasher(*hashes) + h, err := meta4ra.NewHasher(*hashes) + if err != nil { + log.Fatalln(err) + } h.Start() - br := bufio.NewReaderSize(os.Stdin, 1<<20) - buf := make([]byte, 1<<20) + br := bufio.NewReaderSize(os.Stdin, meta4ra.BufLen) + buf := make([]byte, meta4ra.BufLen) size, err := io.CopyBuffer(h, br, buf) if err != nil { log.Fatalln(err) } + dgsts, err := h.Sums() + if err != nil { + log.Fatalln(err) + } f := meta4ra.File{ Name: path.Base(*fn), Description: *desc, Size: uint64(size), URLs: urls, - Hashes: h.Sums(), + Hashes: dgsts, } if *sigPGP != "" { sigData, err := os.ReadFile(*sigPGP) diff --git a/cmd/meta4ra/hash.go b/cmd/meta4ra/hash.go new file mode 100644 index 0000000..266aa61 --- /dev/null +++ b/cmd/meta4ra/hash.go @@ -0,0 +1,63 @@ +// meta4ra -- Metalink 4.0 utilities +// Copyright (C) 2021-2024 Sergey Matveev +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, version 3 of the License. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +package main + +import ( + "bufio" + "flag" + "fmt" + "io" + "log" + "os" + "strings" + + meta4ra "go.stargrave.org/meta4ra/internal" +) + +func runHash() { + hashes := flag.String("hashes", meta4ra.HashesDefault, + "hash-name:commandline[,...]") + flag.Usage = func() { + fmt.Fprintf(flag.CommandLine.Output(), + "Usage: %s [-hashes ...] < data | read hash name\n", os.Args[0]) + flag.PrintDefaults() + fmt.Fprint(flag.CommandLine.Output(), ` +Only the first hash from -hashes will be used. +`) + } + flag.Parse() + + hsh := *hashes + if i := strings.Index(hsh, ","); i != -1 { + hsh = hsh[:i] + } + hasher, err := meta4ra.NewHasher(hsh) + if err != nil { + log.Fatalln(err) + } + if err = hasher.Start(); err != nil { + log.Fatalln(err) + } + if _, err = io.Copy(hasher, bufio.NewReaderSize( + os.Stdin, meta4ra.BufLen)); err != nil { + log.Fatalln(err) + } + sums, err := hasher.Sums() + if err != nil { + log.Fatalln(err) + } + fmt.Println(sums[0].Hash, sums[0].Type) +} diff --git a/cmd/meta4ra/main.go b/cmd/meta4ra/main.go new file mode 100644 index 0000000..334ddc8 --- /dev/null +++ b/cmd/meta4ra/main.go @@ -0,0 +1,21 @@ +package main + +import ( + "log" + "os" + "path" +) + +func main() { + log.SetFlags(log.Lshortfile) + switch path.Base(os.Args[0]) { + case "meta4ra-check": + runCheck() + case "meta4ra-create": + runCreate() + case "meta4ra-hash": + runHash() + default: + log.Fatalln("unknown command linked") + } +} diff --git a/contrib/mk-meta4 b/contrib/mk-meta4 new file mode 100755 index 0000000..b95a718 --- /dev/null +++ b/contrib/mk-meta4 @@ -0,0 +1,12 @@ +#!/bin/sh -e + +ext=meta4 +opts="$@" +hashes="${META4RA_HASHES:-`meta4ra-hashes-detect`}" +find . -type f -maxdepth 1 -not -name "*.$ext" | while read f ; do + f="${f#./}" + [ "$f" != README ] || continue + [ ! -s "$f.$ext" ] || continue + pv --wait --name "$f" "$f" | + meta4ra-create $opts -hashes "$hashes" -fn "$f" -mtime "$f" > "$f".$ext +done diff --git a/contrib/shake128sum b/contrib/shake128sum deleted file mode 100755 index 5d4ec85..0000000 --- a/contrib/shake128sum +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh -e -# sha3sum is expected to be from p5-Digest-SHA3 - -sha3sum -a 128000 | ( dd bs=1 count=64 2>/dev/null ; printf "\n" ) diff --git a/contrib/shake256sum b/contrib/shake256sum deleted file mode 100755 index d02f597..0000000 --- a/contrib/shake256sum +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh -e -# sha3sum is expected to be from p5-Digest-SHA3 - -sha3sum -a 256000 | ( dd bs=1 count=128 2>/dev/null ; printf "\n" ) diff --git a/contrib/streebog256sum b/contrib/streebog256sum deleted file mode 100755 index 1e67e33..0000000 --- a/contrib/streebog256sum +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -nettle-hash --algorithm=streebog256 --raw | xxd -c 0 -p diff --git a/contrib/streebog512sum b/contrib/streebog512sum deleted file mode 100755 index a7ab24d..0000000 --- a/contrib/streebog512sum +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -nettle-hash --algorithm=streebog512 --raw | xxd -c 0 -p diff --git a/go.mod b/go.mod index 47276eb..4aef9b2 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,16 @@ module go.stargrave.org/meta4ra go 1.20 + +require ( + github.com/dchest/skein v0.0.0-20171112102903-d7f1022db390 + github.com/zeebo/xxh3 v1.0.2 + go.cypherpunks.ru/gogost/v5 v5.14.0 + golang.org/x/crypto v0.21.0 + lukechampine.com/blake3 v1.2.1 +) + +require ( + github.com/klauspost/cpuid/v2 v2.0.9 // indirect + golang.org/x/sys v0.18.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..e703b36 --- /dev/null +++ b/go.sum @@ -0,0 +1,15 @@ +github.com/dchest/skein v0.0.0-20171112102903-d7f1022db390 h1:oNcAGoFeaPCgOnlARnJMQqgoq1UMlGwW7PFJddtTF2c= +github.com/dchest/skein v0.0.0-20171112102903-d7f1022db390/go.mod h1:sh8l6PI4IHMaBmo2rlnHxnJDjXY7rxmDeaGSyupxMVM= +github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= +github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= +go.cypherpunks.ru/gogost/v5 v5.14.0 h1:61jW5GfETolg2N1uoMnFzQ3l8HRI3RUFVsu4VHak4wk= +go.cypherpunks.ru/gogost/v5 v5.14.0/go.mod h1:U5P9FOGW0iJK3A8X+yArg+2drPxo3Qvv9f97haBoSZo= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI= +lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= diff --git a/hasher.go b/hasher.go deleted file mode 100644 index 135f040..0000000 --- a/hasher.go +++ /dev/null @@ -1,108 +0,0 @@ -// meta4ra -- Metalink 4.0 utilities -// Copyright (C) 2021-2024 Sergey Matveev -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, version 3 of the License. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -package meta4ra - -import ( - "bytes" - "io" - "log" - "os/exec" - "strings" - "sync" -) - -// Sorted by preference order. -var HashesDefault = []string{ - "blake3-256:b3sum", - "skein-512:skein512", - "skein-256:skein256", - "shake128:goshake128", - "shake256:goshake256", - "sha-512:sha512", - "sha-256:sha256", - "streebog-256:streebog256sum", - "streebog-512:streebog512sum", -} - -type Hasher struct { - Names []string - Cmds []*exec.Cmd - Ins []io.WriteCloser - Outs []io.ReadCloser - wg sync.WaitGroup -} - -func NewHasher(hashes string) *Hasher { - h := Hasher{} - for _, hc := range strings.Split(hashes, ",") { - cols := strings.SplitN(hc, ":", 2) - name, cmdline := cols[0], cols[1] - cmd := exec.Command(cmdline) - in, err := cmd.StdinPipe() - if err != nil { - log.Fatalln(err) - } - out, err := cmd.StdoutPipe() - if err != nil { - log.Fatalln(err) - } - h.Names = append(h.Names, name) - h.Ins = append(h.Ins, in) - h.Outs = append(h.Outs, out) - h.Cmds = append(h.Cmds, cmd) - } - return &h -} - -func (h *Hasher) Start() { - for _, cmd := range h.Cmds { - if err := cmd.Start(); err != nil { - log.Fatalln(err) - } - } -} - -func (h *Hasher) Write(p []byte) (n int, err error) { - h.wg.Add(len(h.Names)) - for _, in := range h.Ins { - go func(in io.WriteCloser) { - if _, err := io.Copy(in, bytes.NewReader(p)); err != nil { - log.Fatalln(err) - } - h.wg.Done() - }(in) - } - h.wg.Wait() - return len(p), nil -} - -func (h *Hasher) Sums() []Hash { - sums := make([]Hash, 0, len(h.Names)) - for i, name := range h.Names { - if err := h.Ins[i].Close(); err != nil { - log.Fatalln(err) - } - dgst, err := io.ReadAll(h.Outs[i]) - if err != nil { - log.Fatalln(err) - } - sums = append(sums, Hash{Type: name, Hash: string(dgst[:len(dgst)-1])}) - if err = h.Cmds[i].Wait(); err != nil { - log.Fatalln(err) - } - } - return sums -} diff --git a/internal/builtin.go b/internal/builtin.go new file mode 100644 index 0000000..a65e889 --- /dev/null +++ b/internal/builtin.go @@ -0,0 +1,59 @@ +// meta4ra -- Metalink 4.0 utilities +// Copyright (C) 2021-2024 Sergey Matveev +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, version 3 of the License. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +package internal + +import ( + "crypto/sha256" + "crypto/sha512" + "encoding/hex" + "hash" + "io" +) + +const BuiltinCmd = "builtin" + +var ( + BuiltinHashes map[string]func() hash.Hash = map[string]func() hash.Hash{ + "sha-256": sha256.New, + "sha-512": sha512.New, + } + HashesDefault = "sha-512:builtin,sha-256:builtin" +) + +type BuiltinHasher struct { + h hash.Hash + finished bool +} + +func (h *BuiltinHasher) Write(p []byte) (int, error) { + return h.h.Write(p) +} + +func (h *BuiltinHasher) Read(p []byte) (int, error) { + if h.finished { + return 0, io.EOF + } + if len(p) < 2*h.h.Size() { + panic("too small buffer for BuiltinHasher.h.Sum()") + } + hex.Encode(p, h.h.Sum(nil)) + h.finished = true + return hex.EncodedLen(h.h.Size()), nil +} + +func (h *BuiltinHasher) Close() error { + return nil +} diff --git a/common.go b/internal/common.go similarity index 91% rename from common.go rename to internal/common.go index 5868c2c..0418574 100644 --- a/common.go +++ b/internal/common.go @@ -14,10 +14,11 @@ // along with this program. If not, see . // Metalink 4.0 utilities -package meta4ra +package internal const ( - Generator = "meta4ra/0.6.0" + Generator = "meta4ra/0.7.0" SigMediaTypePGP = "application/pgp-signature" SigMediaTypeSSH = "application/ssh-signature" + BufLen = 1 << 20 ) diff --git a/internal/hasher.go b/internal/hasher.go new file mode 100644 index 0000000..da54b7d --- /dev/null +++ b/internal/hasher.go @@ -0,0 +1,127 @@ +// meta4ra -- Metalink 4.0 utilities +// Copyright (C) 2021-2024 Sergey Matveev +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, version 3 of the License. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +package internal + +import ( + "bytes" + "errors" + "io" + "os/exec" + "strings" +) + +type Hasher struct { + Names []string + Cmds []*exec.Cmd + Ins []io.WriteCloser + Outs []io.ReadCloser +} + +func NewHasher(hashes string) (*Hasher, error) { + h := Hasher{} + for _, hc := range strings.Split(hashes, ",") { + cols := strings.SplitN(hc, ":", 2) + name, cmdline := cols[0], cols[1] + if cmdline == BuiltinCmd { + newHash, exists := BuiltinHashes[name] + if !exists { + return nil, errors.New("no builtin hash: " + name) + } + b := &BuiltinHasher{h: newHash()} + h.Names = append(h.Names, name) + h.Ins = append(h.Ins, b) + h.Outs = append(h.Outs, b) + h.Cmds = append(h.Cmds, nil) + } else { + cmd := exec.Command("/bin/sh", "-e", "-c", cmdline) + in, err := cmd.StdinPipe() + if err != nil { + return &h, err + } + out, err := cmd.StdoutPipe() + if err != nil { + return &h, err + } + h.Names = append(h.Names, name) + h.Ins = append(h.Ins, in) + h.Outs = append(h.Outs, out) + h.Cmds = append(h.Cmds, cmd) + } + } + return &h, nil +} + +func (h *Hasher) Start() (err error) { + for _, cmd := range h.Cmds { + if cmd == nil { + continue + } + err = cmd.Start() + if err != nil { + return + } + } + return nil +} + +func (h *Hasher) Stop() { + for _, cmd := range h.Cmds { + if cmd == nil || cmd.Process == nil { + continue + } + cmd.Process.Kill() + cmd.Process.Release() + } +} + +func (h *Hasher) Write(p []byte) (n int, rerr error) { + errs := make(chan error) + for _, in := range h.Ins { + go func(in io.WriteCloser) { + _, err := io.Copy(in, bytes.NewReader(p)) + errs <- err + }(in) + } + for i := 0; i < len(h.Names); i++ { + if err := <-errs; err != nil { + rerr = err + } + } + n = len(p) + return +} + +func (h *Hasher) Sums() (sums []Hash, err error) { + sums = make([]Hash, 0, len(h.Names)) + for i, name := range h.Names { + if err = h.Ins[i].Close(); err != nil { + return + } + var out []byte + out, err = io.ReadAll(h.Outs[i]) + if err != nil { + return + } + if cmd := h.Cmds[i]; cmd != nil { + if err = h.Cmds[i].Wait(); err != nil { + return + } + } + cols := strings.Fields(strings.TrimSuffix(string(out), "\n")) + sums = append(sums, Hash{Type: name, Hash: cols[0]}) + } + return +} diff --git a/scheme.go b/internal/scheme.go similarity index 99% rename from scheme.go rename to internal/scheme.go index a1cbec3..9d0ba00 100644 --- a/scheme.go +++ b/internal/scheme.go @@ -13,7 +13,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -package meta4ra +package internal import ( "encoding/xml" diff --git a/internal/thirdparty.go b/internal/thirdparty.go new file mode 100644 index 0000000..9c1b726 --- /dev/null +++ b/internal/thirdparty.go @@ -0,0 +1,95 @@ +//go:build thirdparty + +// meta4ra -- Metalink 4.0 utilities +// Copyright (C) 2021-2024 Sergey Matveev +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, version 3 of the License. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +package internal + +import ( + "hash" + + "github.com/dchest/skein" + "go.cypherpunks.ru/gogost/v5/gost34112012256" + "go.cypherpunks.ru/gogost/v5/gost34112012512" + "golang.org/x/crypto/blake2b" + "golang.org/x/crypto/sha3" + "lukechampine.com/blake3" +) + +func init() { + // Those are better than SHA2, prepend them + name := "shake256" + BuiltinHashes[name] = func() hash.Hash { + return sha3.NewShake256() + } + HashesDefault = name + ":builtin," + HashesDefault + + name = "shake128" + BuiltinHashes[name] = func() hash.Hash { + return sha3.NewShake128() + } + HashesDefault = name + ":builtin," + HashesDefault + + name = "skein-512" + BuiltinHashes[name] = func() hash.Hash { + return skein.NewHash(64) + } + HashesDefault = name + ":builtin," + HashesDefault + + name = "blake2b-256" + BuiltinHashes[name] = func() hash.Hash { + h, err := blake2b.New(32, nil) + if err != nil { + panic(err) + } + return h + } + HashesDefault = name + ":builtin," + HashesDefault + + name = "blake2b-512" + BuiltinHashes[name] = func() hash.Hash { + h, err := blake2b.New(64, nil) + if err != nil { + panic(err) + } + return h + } + HashesDefault = name + ":builtin," + HashesDefault + + name = "blake3-256" + BuiltinHashes[name] = func() hash.Hash { + return blake3.New(32, nil) + } + HashesDefault = name + ":builtin," + HashesDefault + + // Those are slower than SHA2, append them + name = "streebog-512" + BuiltinHashes[name] = func() hash.Hash { + return gost34112012512.New() + } + HashesDefault = HashesDefault + "," + name + ":builtin" + + name = "streebog-256" + BuiltinHashes[name] = func() hash.Hash { + return gost34112012256.New() + } + HashesDefault = HashesDefault + "," + name + ":builtin" + + name = "xxh3-128" + BuiltinHashes[name] = func() hash.Hash { + return NewXXH3128() + } + HashesDefault = HashesDefault + "," + name + ":builtin" +} diff --git a/internal/xxh3.go b/internal/xxh3.go new file mode 100644 index 0000000..6f3f9d8 --- /dev/null +++ b/internal/xxh3.go @@ -0,0 +1,26 @@ +//go:build thirdparty + +package internal + +import ( + "hash" + + "github.com/zeebo/xxh3" +) + +type XXH3128 struct { + *xxh3.Hasher +} + +func (x XXH3128) Size() int { + return 16 +} + +func (x XXH3128) Sum(b []byte) []byte { + tmp := x.Hasher.Sum128().Bytes() + return append(b, tmp[:]...) +} + +func NewXXH3128() hash.Hash { + return &XXH3128{xxh3.New()} +} -- 2.44.0