From: Sergey Matveev Date: Thu, 4 May 2023 10:00:26 +0000 (+0300) Subject: meta4-check workability again X-Git-Tag: v0.1.0~3 X-Git-Url: http://www.git.stargrave.org/?p=meta4ra.git;a=commitdiff_plain;h=443182b338389120eb9039e27fe49f7b28cc5419 meta4-check workability again --- diff --git a/README b/README index ee95c19..45c90ba 100644 --- a/README +++ b/README @@ -17,16 +17,18 @@ meta4-create utility is used to create Metalink4 Following hashes are predefined by default: * sha-256:sha256 -- standardized name; standard command in many OSes - Maybe fast on hardware accelerated CPUs + May be fast on hardware accelerated CPUs. * sha-512:sha512 -- standardized name; standard command in many OSes - Faster on 64-bit CPUs than software sha-256 + Faster on 64-bit CPUs than software sha-256. * skein-256:skein256 -- non-standardized name; out-of-box command in FreeBSD - Faster than software sha-*/shake* + Faster than software sha-*/shake*. * skein-512:skein512 -- non-standardized name; out-of-box command in FreeBSD - Faster on 64-bit CPUs than skein-256 -* shake128:shake128sum -- standardized name; command is in contrib/ - Can be faster on hardware than sha-*/skein-* -* shake256:shake256sum -- standardized name; command is in contrib/ + Faster on 64-bit CPUs than skein-256. +* shake128:goshake128 -- standardized name; non-standard command + Faster than software sha-*. Much faster on hardware. +* shake256:goshake256 -- standardized name; non-standard command + Same speed as sha-512 on 64-bit CPUs. + Much faster on hardware. * streebog-256:streebog256sum -- non-standardized name; command is in contrib/ * streebog-512:streebog512sum -- non-standardized name; command is in contrib/ * blake3-256:b3sum -- non-standardized name; additional package in most OSes @@ -36,3 +38,9 @@ SHA2 and SHA3 (SHAKE*) are USA's NIST standards. Streebog is Russian Federation's government standard. Skein is SHA3-finalist. BLAKE3 is reduced round Merklee-tree-based BLAKE2 descendant. BLAKE2 is reduced round BLAKE, that also was one of SHA3-finalists. + +meta4-check utility is used to check files against provided Metalink4. + +-extract-sig -- store the OpenPGP signature (if exists) in .asc file nearby. +-hashes -- same as in meta4-create. Hashes are sorted by preferences + priority. First matching one will be used to check integrity. diff --git a/cmd/meta4-check/main.go b/cmd/meta4-check/main.go index 515c327..45b08bd 100644 --- a/cmd/meta4-check/main.go +++ b/cmd/meta4-check/main.go @@ -20,27 +20,23 @@ package main import ( "bufio" - "crypto/sha256" - "crypto/sha512" - "encoding/hex" "encoding/xml" "flag" "fmt" - "hash" "io" "io/fs" "log" "os" + "strings" "go.stargrave.org/meta4ra" ) func main() { + hashes := flag.String("hashes", strings.Join(meta4ra.HashesDefault, ","), "hash-name:command-s") extractSig := flag.Bool("extract-sig", false, "Extract signature files") log.SetFlags(log.Lshortfile) flag.Parse() - sha256Hasher := sha256.New() - sha512Hasher := sha512.New() bad := false for _, metaPath := range flag.Args() { data, err := os.ReadFile(metaPath) @@ -63,53 +59,45 @@ func main() { bad = true } } - var sha256Digest string - var sha512Digest string - fd, err := os.Open(f.Name) - if err != nil { - continue - } - for _, h := range f.Hashes { - switch h.Type { - case meta4ra.HashSHA256: - sha256Digest = h.Hash - case meta4ra.HashSHA512: - sha512Digest = h.Hash + hasher := meta4ra.NewHasher(*hashes) + var hashTheir string + var hashName string + for i, name := range hasher.Names { + for _, h := range f.Hashes { + if h.Type == name { + hasher.Names = []string{name} + hasher.Cmds = append(hasher.Cmds[:0], hasher.Cmds[i]) + hasher.Ins = append(hasher.Ins[:0], hasher.Ins[i]) + hasher.Outs = append(hasher.Outs[:0], hasher.Outs[i]) + hashName = name + hashTheir = h.Hash + goto HashFound + } } } - var digestTheir string - var digestName string - var hasher hash.Hash - if sha512Digest != "" { - digestName = meta4ra.HashSHA512 - digestTheir = sha512Digest - hasher = sha512Hasher - } else if sha256Digest != "" { - digestName = meta4ra.HashSHA256 - digestTheir = sha256Digest - hasher = sha256Hasher - } else { - fd.Close() - fmt.Println("Error:", f.Name, "no satisfiable hash algorithm found") - bad = true + log.Fatalln("no common hashes found for:", f.Name) + HashFound: + fd, err := os.Open(f.Name) + if err != nil { continue } - hasher.Reset() - _, err = io.Copy(hasher, bufio.NewReader(fd)) + hasher.Start() + _, err = io.Copy(hasher, bufio.NewReaderSize(fd, 1<<20)) fd.Close() + sums := hasher.Sums() if err != nil { fmt.Println("Error:", f.Name, err) bad = true continue } - digestOur := hex.EncodeToString(hasher.Sum(nil)) - if digestOur == digestTheir { - fmt.Println(f.Name, digestName, "good") + hashOur := sums[0].Hash + if hashOur == hashTheir { + fmt.Println(f.Name, hashName, "good") } else { fmt.Println( - "Hash does not match:", f.Name, digestName, - "our:", digestOur, - "their:", digestTheir, + "Hash does not match:", f.Name, hashName, + "our:", hashOur, + "their:", hashTheir, ) bad = true continue diff --git a/cmd/meta4-create/main.go b/cmd/meta4-create/main.go index b5d9aba..f5d8b5c 100644 --- a/cmd/meta4-create/main.go +++ b/cmd/meta4-create/main.go @@ -20,103 +20,24 @@ package main import ( "bufio" - "bytes" "encoding/xml" "flag" "io" "log" "os" - "os/exec" "path" "strings" - "sync" "time" "go.stargrave.org/meta4ra" ) -type Hasher struct { - names []string - cmds []*exec.Cmd - ins []io.WriteCloser - outs []io.ReadCloser - wg sync.WaitGroup -} - -func NewHasher(hashes string) *Hasher { - h := Hasher{} - for _, hc := range strings.Split(hashes, ",") { - cols := strings.SplitN(hc, ":", 2) - name, cmdline := cols[0], cols[1] - cmd := exec.Command(cmdline) - in, err := cmd.StdinPipe() - if err != nil { - log.Fatalln(err) - } - out, err := cmd.StdoutPipe() - if err != nil { - log.Fatalln(err) - } - if err = cmd.Start(); err != nil { - log.Fatalln(err) - } - h.names = append(h.names, name) - h.ins = append(h.ins, in) - h.outs = append(h.outs, out) - h.cmds = append(h.cmds, cmd) - } - return &h -} - -func (h *Hasher) Write(p []byte) (n int, err error) { - h.wg.Add(len(h.names)) - for _, in := range h.ins { - go func(in io.WriteCloser) { - if _, err := io.Copy(in, bytes.NewReader(p)); err != nil { - log.Fatalln(err) - } - h.wg.Done() - }(in) - } - h.wg.Wait() - return len(p), nil -} - -func (h *Hasher) Sums() []meta4ra.Hash { - sums := make([]meta4ra.Hash, 0, len(h.names)) - for i, name := range h.names { - if err := h.ins[i].Close(); err != nil { - log.Fatalln(err) - } - dgst, err := io.ReadAll(h.outs[i]) - if err != nil { - log.Fatalln(err) - } - sums = append(sums, meta4ra.Hash{Type: name, Hash: string(dgst[:len(dgst)-1])}) - if err = h.cmds[i].Wait(); err != nil { - log.Fatalln(err) - } - } - return sums -} - func main() { fn := flag.String("fn", "", "Filename") mtime := flag.String("mtime", "", "Take that file's mtime as a Published date") desc := flag.String("desc", "", "Description") sig := flag.String("sig", "", "Path to signature file") - hashesDef := []string{ - "sha-256:sha256", - "sha-512:sha512", - "shake128:shake128sum", - "shake256:shake256sum", - "streebog-256:streebog256sum", - "streebog-512:streebog512sum", - "skein-256:skein256", - "skein-512:skein512", - "blake3-256:b3sum", - } - hashes := flag.String("hashes", strings.Join(hashesDef, ","), "hash-name:command-s") + hashes := flag.String("hashes", strings.Join(meta4ra.HashesDefault, ","), "hash-name:command-s") torrent := flag.String("torrent", "", "Torrent URL") log.SetFlags(log.Lshortfile) flag.Parse() @@ -127,7 +48,8 @@ func main() { for _, u := range flag.Args() { urls = append(urls, meta4ra.URL{URL: u}) } - h := NewHasher(*hashes) + h := meta4ra.NewHasher(*hashes) + h.Start() br := bufio.NewReaderSize(os.Stdin, 1<<20) buf := make([]byte, 1<<20) size, err := io.CopyBuffer(h, br, buf) diff --git a/common.go b/common.go index 0c56faf..a20c243 100644 --- a/common.go +++ b/common.go @@ -18,53 +18,7 @@ along with this program. If not, see . // Metalink 4.0 utilities package meta4ra -import ( - "encoding/xml" - "time" -) - const ( Generator = "meta4ra/0.3.0" GPGSigMediaType = "application/pgp-signature" ) - -type Metalink struct { - XMLName xml.Name `xml:"urn:ietf:params:xml:ns:metalink metalink"` - Files []File `xml:"file"` - Generator string `xml:"generator,,omitempty"` - Published time.Time `xml:"published,,omitempty"` -} - -type File struct { - XMLName xml.Name `xml:"file"` - Name string `xml:"name,attr"` - Description string `xml:"description,,omitempty"` - Hashes []Hash `xml:"hash,,omitempty"` - MetaURLs []MetaURL `xml:"metaurl,,omitempty"` - Signature *Signature `xml:"signature"` - Size uint64 `xml:"size,,omitempty"` - URLs []URL `xml:"url,,omitempty"` -} - -type URL struct { - XMLName xml.Name `xml:"url"` - URL string `xml:",chardata"` -} - -type Signature struct { - XMLName xml.Name `xml:"signature"` - MediaType string `xml:"mediatype,attr"` - Signature string `xml:",cdata"` -} - -type Hash struct { - XMLName xml.Name `xml:"hash"` - Type string `xml:"type,attr"` - Hash string `xml:",chardata"` -} - -type MetaURL struct { - XMLName xml.Name `xml:"metaurl"` - MediaType string `xml:"mediatype,attr"` - URL string `xml:",chardata"` -} diff --git a/hasher.go b/hasher.go new file mode 100644 index 0000000..50c3d84 --- /dev/null +++ b/hasher.go @@ -0,0 +1,93 @@ +package meta4ra + +import ( + "bytes" + "io" + "log" + "os/exec" + "strings" + "sync" +) + +// Sorted by preference order. +var HashesDefault = []string{ + "blake3-256:b3sum", + "skein-512:skein512", + "skein-256:skein256", + "shake128:goshake128", + "shake256:goshake256", + "sha-512:sha512", + "sha-256:sha256", + "streebog-256:streebog256sum", + "streebog-512:streebog512sum", +} + +type Hasher struct { + Names []string + Cmds []*exec.Cmd + Ins []io.WriteCloser + Outs []io.ReadCloser + wg sync.WaitGroup +} + +func NewHasher(hashes string) *Hasher { + h := Hasher{} + for _, hc := range strings.Split(hashes, ",") { + cols := strings.SplitN(hc, ":", 2) + name, cmdline := cols[0], cols[1] + cmd := exec.Command(cmdline) + in, err := cmd.StdinPipe() + if err != nil { + log.Fatalln(err) + } + out, err := cmd.StdoutPipe() + if err != nil { + log.Fatalln(err) + } + h.Names = append(h.Names, name) + h.Ins = append(h.Ins, in) + h.Outs = append(h.Outs, out) + h.Cmds = append(h.Cmds, cmd) + } + return &h +} + +func (h *Hasher) Start() { + for _, cmd := range h.Cmds { + if err := cmd.Start(); err != nil { + log.Fatalln(err) + } + } +} + +func (h *Hasher) Write(p []byte) (n int, err error) { + h.wg.Add(len(h.Names)) + for _, in := range h.Ins { + go func(in io.WriteCloser) { + if _, err := io.Copy(in, bytes.NewReader(p)); err != nil { + log.Fatalln(err) + } + h.wg.Done() + }(in) + } + h.wg.Wait() + return len(p), nil +} + +func (h *Hasher) Sums() []Hash { + sums := make([]Hash, 0, len(h.Names)) + for i, name := range h.Names { + if err := h.Ins[i].Close(); err != nil { + log.Fatalln(err) + } + dgst, err := io.ReadAll(h.Outs[i]) + if err != nil { + log.Fatalln(err) + } + sums = append(sums, Hash{Type: name, Hash: string(dgst[:len(dgst)-1])}) + if err = h.Cmds[i].Wait(); err != nil { + log.Fatalln(err) + } + } + return sums +} diff --git a/scheme.go b/scheme.go new file mode 100644 index 0000000..b839282 --- /dev/null +++ b/scheme.go @@ -0,0 +1,64 @@ +/* +meta4a -- Metalink 4.0 creator +Copyright (C) 2021-2023 Sergey Matveev + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, version 3 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package meta4ra + +import ( + "encoding/xml" + "time" +) + +type Metalink struct { + XMLName xml.Name `xml:"urn:ietf:params:xml:ns:metalink metalink"` + Files []File `xml:"file"` + Generator string `xml:"generator,,omitempty"` + Published time.Time `xml:"published,,omitempty"` +} + +type File struct { + XMLName xml.Name `xml:"file"` + Name string `xml:"name,attr"` + Description string `xml:"description,,omitempty"` + Hashes []Hash `xml:"hash,,omitempty"` + MetaURLs []MetaURL `xml:"metaurl,,omitempty"` + Signature *Signature `xml:"signature"` + Size uint64 `xml:"size,,omitempty"` + URLs []URL `xml:"url,,omitempty"` +} + +type URL struct { + XMLName xml.Name `xml:"url"` + URL string `xml:",chardata"` +} + +type Signature struct { + XMLName xml.Name `xml:"signature"` + MediaType string `xml:"mediatype,attr"` + Signature string `xml:",cdata"` +} + +type Hash struct { + XMLName xml.Name `xml:"hash"` + Type string `xml:"type,attr"` + Hash string `xml:",chardata"` +} + +type MetaURL struct { + XMLName xml.Name `xml:"metaurl"` + MediaType string `xml:"mediatype,attr"` + URL string `xml:",chardata"` +}