From: Sergey Matveev Date: Sat, 14 Mar 2026 10:09:02 +0000 (+0300) Subject: Multiline hashers specification X-Git-Tag: v2.0.0^0 X-Git-Url: http://www.git.stargrave.org/?a=commitdiff_plain;h=43acd9d6a9ccc4933393227fea93446298cc1623;p=meta4ra.git Multiline hashers specification --- diff --git a/HASHES b/HASHES index 52445e7..cbee6b3 100644 --- a/HASHES +++ b/HASHES @@ -6,20 +6,23 @@ of the hashes. Various utilities, various versions, various OS distributions have different set of available/better options. That is why meta4ra-create -and meta4ra-check utilities have -hashes option, where you specify set +and meta4ra-check utilities have -hashers option, where you specify set of supported algorithms and command line to run for their calculation. --hashes is comma-separated list of colon-separated "name:cmdline" pairs. - -hashes "skein-512:skein512,sha512:libressl dgst -sha512" +-hashers is newline-separated list of "name cmdline" pairs. + -hashers " + skein-512 skein512 + sha512 libressl dgst -sha512 + " option tells, that for calculation of Skein-512 you have to run skein512 command, and for SHA2-512 "libressl ..." one. They are invoked under "/bin/sh -e -c" command, so pipelines are also allowable there. Data is fed to their stdout and they are expected to print hash value in hexadecimal form as a first (or single) column to stdout. First found common algorithm is used by default for file verification in -meta4ra-check utility, so order in -hashes is important. +meta4ra-check utility, so order in -hashers is important. -meta4ra-hashes-detect utility conveniently checks various predefined -known commands and outputs -hashes-compatible string for your system. +meta4ra-hashers-detect utility conveniently checks various predefined +known commands and outputs -hashers-compatible string for your system. If you use "builtin" word as a command, then builtin implementation of the hash will be used. By default, meta4ra does not require any diff --git a/bin/meta4ra-hashers-detect b/bin/meta4ra-hashers-detect new file mode 100755 index 0000000..66bcb1b --- /dev/null +++ b/bin/meta4ra-hashers-detect @@ -0,0 +1,122 @@ +#!/bin/sh -e +# Detect possible hashers + +desired="$1" + +check() { + local name="$1" + local cmd="$2" + our=$(echo -n hello world | + sh -e -c "$cmd" | + { read h rem ; printf %s "$h"; }) + [ $HSH == "$our" ] && echo "$name $cmd" || return 1 +} + +no() { + echo no $1 >&2 +} + +n=blake3-256 +if [ -z "$desired" ] || [ $desired = "$n" ] ; then + HSH=d74981efa70a0c880b8d8c1985d075dbcbf679b99a5f9914e5aaf96b831a9e24 + check $n b3sum || # https://github.com/BLAKE3-team/BLAKE3 + check $n "meta4ra-hash -hashers $n:builtin" || + no $n +fi + +n=blake2b-512 +if [ -z "$desired" ] || [ $desired = "$n" ] ; then + HSH=021ced8799296ceca557832ab941a50b4a11f83478cf141f51f933f653ab9fbcc05a037cddbed06e309bf334942c4e58cdf1a46e237911ccd7fcf9787cbc7fd0 + check $n b2sum || # https://blake2.net/, GNU Coreutils + check $n "openssl blake2b512 -r" || + check $n "meta4ra-hash -hashers $n:builtin" || + no $n +fi + +n=blake2b-256 +if [ -z "$desired" ] || [ $desired = "$n" ] ; then + HSH=256c83b297114d201b30179f3f0ef0cace9783622da5974326b436178aeef610 + check $n "b2sum -l 256" || + check $n "meta4ra-hash -hashers $n:builtin" || + no $n +fi + +n=skein-512 +if [ -z "$desired" ] || [ $desired = "$n" ] ; then + HSH=8b4830244fc36daa11177311dc6bf7636376180dce2d29193335878142e7d6f5e9016beba729e0a353dd2fd421c8b2022ee8927f0bce6b88631bb01be2e0f5ba + check $n skein512 || # FreeBSD's out-of-box command + no $n +fi + +n=shake128 +if [ -z "$desired" ] || [ $desired = "$n" ] ; then + HSH=3a9159f071e4dd1c8c4f968607c30942e120d8156b8b1e72e0d376e8871cb8b8 + check $n goshake128 || # go.stargrave.org/gosha3 + check $n "sha3sum -a 128000 | dd bs=1 count=64 2>/dev/null ; echo" || # p5-Digest-SHA3 + check $n "meta4ra-hash -hashers $n:builtin" || + no $n + # openssl shake128 -- useless, as it outputs only 128 bits +fi + +n=shake256 +if [ -z "$desired" ] || [ $desired = "$n" ] ; then + HSH=369771bb2cb9d2b04c1d54cca487e372d9f187f73f7ba3f65b95c8ee7798c527f4f3c2d55c2d46a29f2e945d469c3df27853a8735271f5cc2d9e889544357116 + check $n goshake256 || + check $n "sha3sum -a 256000 | dd bs=1 count=128 2>/dev/null ; echo" || + check $n "meta4ra-hash -hashers $n:builtin" || + no $n +fi + +n=sha-512 +if [ -z "$desired" ] || [ $desired = "$n" ] ; then + HSH=309ecc489c12d6eb4cc40f50c902f2b4d0ed77ee511a7c7a9bcd3ca86d4cd86f989dd35bc5ff499670da34255b45b0cfd830e81f605dcf7dc5542e93ae9cd76f + check $n sha512 || + check $n sha512sum || + check $n "libressl dgst -sha512" || + check $n "openssl sha512 -r" || + check $n "meta4ra-hash -hashers $n:builtin" || + no $n +fi + +n=sha-256 +if [ -z "$desired" ] || [ $desired = "$n" ] ; then + HSH=b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9 + check $n sha256 || + check $n sha256sum || + check $n "libressl dgst -sha256" || + check $n "openssl sha256 -r" || + check $n "meta4ra-hash -hashers $n:builtin" || + no $n +fi + +unxxd="xxd -c 0 -p" +unhexdump="hexdump -v -e '/1 \"%02x\"' ; echo" +n=streebog-512 +if [ -z "$desired" ] || [ $desired = "$n" ] ; then + HSH=84d883ede9fa6ce855d82d8c278ecd9f5fc88bf0602831ae0c38b9b506ea3cb02f3fa076b8f5664adf1ff862c0157da4cc9a83e141b738ff9268a9ba3ed6f563 + check $n streebog512 || # go.cypherpunks.su/gogost + check $n "nettle-hash --algorithm=streebog512 --raw | $unxxd" || + check $n "nettle-hash --algorithm=streebog512 --raw | $unhexdump" || + check $n "libressl dgst -streebog512" || + check $n "meta4ra-hash -hashers $n:builtin" || + no $n +fi + +n=streebog-256 +if [ -z "$desired" ] || [ $desired = "$n" ] ; then + HSH=c600fd9dd049cf8abd2f5b32e840d2cb0e41ea44de1c155dcd88dc84fe58a855 + check $n streebog256 || + check $n "nettle-hash --algorithm=streebog256 --raw | $unxxd" || + check $n "nettle-hash --algorithm=streebog256 --raw | $unhexdump" || + check $n "libressl dgst -streebog256" || + check $n "meta4ra-hash -hashers $n:builtin" || + no $n +fi + +n=xxh3-128 +if [ -z "$desired" ] || [ $desired = "$n" ] ; then + HSH=df8d09e93f874900a99b8775cc15b6c7 + check $n "xxhsum -H128" || # https://cyan4973.github.io/xxHash/ + check $n "meta4ra-hash -hashers $n:builtin" || + no $n +fi diff --git a/bin/meta4ra-hashes-detect b/bin/meta4ra-hashes-detect deleted file mode 100755 index 96f6bae..0000000 --- a/bin/meta4ra-hashes-detect +++ /dev/null @@ -1,83 +0,0 @@ -#!/bin/sh -e -# Autodetect possible commandlines for various hashes. Example usage: -# meta4ra-{create,check} -hashes "$(meta4ra-hashes-detect)" ... - -hw="hello world" -hashes="" - -desired="$1" - -check() { - local name="$1" - if [ -n "$desired" ] && [ $desired != "$name" ] ; then return 1 ; fi - local cmd="$2" - local cmdline="$cmd" - [ "$cmd" != "builtin" ] || cmd="meta4ra-hash -hashes $name:builtin" - our=$(echo -n hello world | sh -e -c "$cmd" | { read h rem ; printf %s "$h"; }) - [ $HSH = "$our" ] && hashes="$hashes,$name:$cmdline" -} - -HSH=d74981efa70a0c880b8d8c1985d075dbcbf679b99a5f9914e5aaf96b831a9e24 -check blake3-256 b3sum || # https://github.com/BLAKE3-team/BLAKE3 -check blake3-256 builtin || : - -HSH=021ced8799296ceca557832ab941a50b4a11f83478cf141f51f933f653ab9fbcc05a037cddbed06e309bf334942c4e58cdf1a46e237911ccd7fcf9787cbc7fd0 -check blake2b-512 b2sum || # https://blake2.net/, GNU Coreutils -check blake2b-512 "openssl blake2b512 -r" || -check blake2b-512 builtin || : - -HSH=256c83b297114d201b30179f3f0ef0cace9783622da5974326b436178aeef610 -check blake2b-256 "b2sum -l 256" || -check blake2b-512 builtin || : - -HSH=8b4830244fc36daa11177311dc6bf7636376180dce2d29193335878142e7d6f5e9016beba729e0a353dd2fd421c8b2022ee8927f0bce6b88631bb01be2e0f5ba -check skein-512 skein512 || # FreeBSD's out-of-box command -check skein-512 builtin || : - -HSH=3a9159f071e4dd1c8c4f968607c30942e120d8156b8b1e72e0d376e8871cb8b8 -check shake128 goshake128 || # go.stargrave.org/gosha3 -check shake128 "sha3sum -a 128000 | dd bs=1 count=64 2>/dev/null" || # p5-Digest-SHA3 -check shake128 builtin || : -# openssl shake128 -- useless, as it outputs only 128 bits - -HSH=369771bb2cb9d2b04c1d54cca487e372d9f187f73f7ba3f65b95c8ee7798c527f4f3c2d55c2d46a29f2e945d469c3df27853a8735271f5cc2d9e889544357116 -check shake256 goshake256 || -check shake256 "sha3sum -a 256000 | dd bs=1 count=128 2>/dev/null" || -check shake256 builtin || : - -HSH=309ecc489c12d6eb4cc40f50c902f2b4d0ed77ee511a7c7a9bcd3ca86d4cd86f989dd35bc5ff499670da34255b45b0cfd830e81f605dcf7dc5542e93ae9cd76f -check sha-512 sha512 || -check sha-512 sha512sum || -check sha-512 "libressl dgst -sha512" || -check sha-512 "openssl sha512 -r" || -check sha-512 builtin || : - -HSH=b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9 -check sha-256 sha256 || -check sha-256 sha256sum || -check sha-256 "libressl dgst -sha256" || -check sha-256 "openssl sha256 -r" || -check sha-256 builtin || : - -unxxd="xxd -c 0 -p" -unhexdump="hexdump -v -e '/1 \"%02x\"' ; echo" -HSH=84d883ede9fa6ce855d82d8c278ecd9f5fc88bf0602831ae0c38b9b506ea3cb02f3fa076b8f5664adf1ff862c0157da4cc9a83e141b738ff9268a9ba3ed6f563 -check streebog-512 streebog512 $hsh || # go.cypherpunks.su/gogost -check streebog-512 "nettle-hash --algorithm=streebog512 --raw | $unxxd" || -check streebog-512 "nettle-hash --algorithm=streebog512 --raw | $unhexdump" || -check streebog-512 "libressl dgst -streebog512" || -check streebog-512 builtin || : - -HSH=c600fd9dd049cf8abd2f5b32e840d2cb0e41ea44de1c155dcd88dc84fe58a855 -check streebog-256 streebog256 || -check streebog-256 "nettle-hash --algorithm=streebog256 --raw | $unxxd" || -check streebog-256 "nettle-hash --algorithm=streebog256 --raw | $unhexdump" || -check streebog-256 "libressl dgst -streebog256" || -check streebog-256 builtin || : - -HSH=df8d09e93f874900a99b8775cc15b6c7 -check xxh3-128 "xxhsum -H128" || # https://cyan4973.github.io/xxHash/ -check xxh3-128 builtin || : - -[ -n "$hashes" ] -echo ${hashes#,} diff --git a/cmd/check.go b/cmd/check.go index 657782b..09bfe2f 100644 --- a/cmd/check.go +++ b/cmd/check.go @@ -36,7 +36,7 @@ func runCheck() { pipe := flag.Bool("pipe", false, "Verify file data from stdin, copy to stdout") progress := flag.Bool("progress", false, "Show progress of piping/downloading") allHashes := flag.Bool("all-hashes", false, "Check all hashes, not the first common one") - hashes := flag.String("hashes", meta4ra.HashesDefault, + hashers := flag.String("hashers", meta4ra.HashersDefault, "hash-name:commandline[,...]") flag.Usage = func() { fmt.Fprintf(flag.CommandLine.Output(), @@ -126,7 +126,7 @@ format, then you can just specify an empty ("") FILE. } } - hasher, err := meta4ra.NewHasher(*hashes) + hasher, err := meta4ra.NewHasher(*hashers) if err != nil { log.Println(f.Name, err) bad = true diff --git a/cmd/create.go b/cmd/create.go index eddd73f..ef72428 100644 --- a/cmd/create.go +++ b/cmd/create.go @@ -40,8 +40,8 @@ func runCreate() { "Path to LibrePGP .asc signature file for inclusion") sigSSH := flag.String("sig-ssh", "", "Path to OpenSSH .sig signature file for inclusion") - hashes := flag.String("hashes", meta4ra.HashesDefault, - "hash-name:commandline[,...]") + hashers := flag.String("hashers", meta4ra.HashersDefault, + "hash-name commandline[\\n...]") noPublished := flag.Bool("no-published", false, "Do not include Published field") noGenerator := flag.Bool("no-generator", false, @@ -78,7 +78,7 @@ will be the priority and "cc" (may be empty) is location. } urls = append(urls, *url) } - h, err := meta4ra.NewHasher(*hashes) + h, err := meta4ra.NewHasher(*hashers) if err != nil { log.Fatal(err) } diff --git a/cmd/hash.go b/cmd/hash.go index 6550d51..7fc377c 100644 --- a/cmd/hash.go +++ b/cmd/hash.go @@ -28,15 +28,15 @@ import ( ) func runHash() { - hashes := flag.String("hashes", meta4ra.HashesDefault, - "hash-name:commandline[,...]") + hashers := flag.String("hashers", meta4ra.HashersDefault, + "hash-name commandline[\\n...]") all := flag.Bool("all", false, "Print all hashes, not the first one") flag.Usage = func() { fmt.Fprintf(flag.CommandLine.Output(), - "Usage: %s [-hashes ...] "$f".$ext + meta4ra-create $opts -hashers "$hashers" -fn "$f" -mtime "$f" >"$f".$ext done diff --git a/internal/builtin.go b/internal/builtin.go index 15fe0d9..bc88c01 100644 --- a/internal/builtin.go +++ b/internal/builtin.go @@ -27,13 +27,17 @@ import ( const BuiltinCmd = "builtin" var ( - BuiltinHashes map[string]func() hash.Hash = map[string]func() hash.Hash{ + BuiltinHashers map[string]func() hash.Hash = map[string]func() hash.Hash{ "sha-256": sha256.New, "sha-512": sha512.New, "shake128": func() hash.Hash { return shake{l: 32, h: sha3.NewSHAKE128()} }, "shake256": func() hash.Hash { return shake{l: 64, h: sha3.NewSHAKE256()} }, } - HashesDefault = "shake128:builtin,shake256:builtin,sha-512:builtin,sha-256:builtin" + HashersDefault = `shake128 builtin +shake256 builtin +sha-512 builtin +sha-256 builtin +` ) type BuiltinHasher struct { diff --git a/internal/common.go b/internal/common.go index 384f7a3..2276101 100644 --- a/internal/common.go +++ b/internal/common.go @@ -21,7 +21,7 @@ import ( ) const ( - Generator = "meta4ra/1.6.0" + Generator = "meta4ra/2.0.0" SigMediaTypePGP = "application/pgp-signature" SigMediaTypeSSH = "application/ssh-signature" BufLen = 1 << 20 diff --git a/internal/hasher.go b/internal/hasher.go index db2b912..31ea1ef 100644 --- a/internal/hasher.go +++ b/internal/hasher.go @@ -30,16 +30,16 @@ type Hasher struct { Outs []io.ReadCloser } -func NewHasher(hashes string) (*Hasher, error) { +func NewHasher(hashers string) (*Hasher, error) { h := Hasher{} - for hc := range strings.SplitSeq(hashes, ",") { + for hc := range strings.SplitSeq(hashers, "\n") { if hc == "" { continue } - cols := strings.SplitN(hc, ":", 2) + cols := strings.SplitN(hc, " ", 2) name, cmdline := cols[0], cols[1] if cmdline == BuiltinCmd { - newHash, exists := BuiltinHashes[name] + newHash, exists := BuiltinHashers[name] if !exists { return nil, errors.New("no builtin hash: " + name) } diff --git a/internal/thirdparty.go b/internal/thirdparty.go index 21a38f7..4a50aa7 100644 --- a/internal/thirdparty.go +++ b/internal/thirdparty.go @@ -29,53 +29,53 @@ import ( func init() { name := "skein-512" - BuiltinHashes[name] = func() hash.Hash { + BuiltinHashers[name] = func() hash.Hash { return skein.NewHash(64) } - HashesDefault = name + ":builtin," + HashesDefault + HashersDefault = name + " builtin\n" + HashersDefault name = "blake2b-256" - BuiltinHashes[name] = func() hash.Hash { + BuiltinHashers[name] = func() hash.Hash { h, err := blake2b.New(32, nil) if err != nil { panic(err) } return h } - HashesDefault = name + ":builtin," + HashesDefault + HashersDefault = name + " builtin\n" + HashersDefault name = "blake2b-512" - BuiltinHashes[name] = func() hash.Hash { + BuiltinHashers[name] = func() hash.Hash { h, err := blake2b.New(64, nil) if err != nil { panic(err) } return h } - HashesDefault = name + ":builtin," + HashesDefault + HashersDefault = name + " builtin\n" + HashersDefault name = "blake3-256" - BuiltinHashes[name] = func() hash.Hash { + BuiltinHashers[name] = func() hash.Hash { return blake3.New(32, nil) } - HashesDefault = name + ":builtin," + HashesDefault + HashersDefault = name + " builtin\n" + HashersDefault // Those are slower than SHA2, append them name = "streebog-512" - BuiltinHashes[name] = func() hash.Hash { + BuiltinHashers[name] = func() hash.Hash { return gost34112012512.New() } - HashesDefault = HashesDefault + "," + name + ":builtin" + HashersDefault = HashersDefault + "\n" + name + " builtin" name = "streebog-256" - BuiltinHashes[name] = func() hash.Hash { + BuiltinHashers[name] = func() hash.Hash { return gost34112012256.New() } - HashesDefault = HashesDefault + "," + name + ":builtin" + HashersDefault = HashersDefault + "\n" + name + " builtin" name = "xxh3-128" - BuiltinHashes[name] = func() hash.Hash { + BuiltinHashers[name] = func() hash.Hash { return NewXXH3128() } - HashesDefault = HashesDefault + "," + name + ":builtin" + HashersDefault = HashersDefault + "\n" + name + " builtin" }