Following hashes are predefined by default:
* sha-256:sha256 -- standardized name; standard command in many OSes
- Maybe fast on hardware accelerated CPUs
+ May be fast on hardware accelerated CPUs.
* sha-512:sha512 -- standardized name; standard command in many OSes
- Faster on 64-bit CPUs than software sha-256
+ Faster on 64-bit CPUs than software sha-256.
* skein-256:skein256 -- non-standardized name; out-of-box command in FreeBSD
- Faster than software sha-*/shake*
+ Faster than software sha-*/shake*.
* skein-512:skein512 -- non-standardized name; out-of-box command in FreeBSD
- Faster on 64-bit CPUs than skein-256
-* shake128:shake128sum -- standardized name; command is in contrib/
- Can be faster on hardware than sha-*/skein-*
-* shake256:shake256sum -- standardized name; command is in contrib/
+ Faster on 64-bit CPUs than skein-256.
+* shake128:goshake128 -- standardized name; non-standard command
+ Faster than software sha-*. Much faster on hardware.
+* shake256:goshake256 -- standardized name; non-standard command
+ Same speed as sha-512 on 64-bit CPUs.
+ Much faster on hardware.
* streebog-256:streebog256sum -- non-standardized name; command is in contrib/
* streebog-512:streebog512sum -- non-standardized name; command is in contrib/
* blake3-256:b3sum -- non-standardized name; additional package in most OSes
Federation's government standard. Skein is SHA3-finalist. BLAKE3 is
reduced round Merklee-tree-based BLAKE2 descendant. BLAKE2 is reduced
round BLAKE, that also was one of SHA3-finalists.
+
+meta4-check utility is used to check files against provided Metalink4.
+
+-extract-sig -- store the OpenPGP signature (if exists) in .asc file nearby.
+-hashes -- same as in meta4-create. Hashes are sorted by preferences
+ priority. First matching one will be used to check integrity.
import (
"bufio"
- "crypto/sha256"
- "crypto/sha512"
- "encoding/hex"
"encoding/xml"
"flag"
"fmt"
- "hash"
"io"
"io/fs"
"log"
"os"
+ "strings"
"go.stargrave.org/meta4ra"
)
func main() {
+ hashes := flag.String("hashes", strings.Join(meta4ra.HashesDefault, ","), "hash-name:command-s")
extractSig := flag.Bool("extract-sig", false, "Extract signature files")
log.SetFlags(log.Lshortfile)
flag.Parse()
- sha256Hasher := sha256.New()
- sha512Hasher := sha512.New()
bad := false
for _, metaPath := range flag.Args() {
data, err := os.ReadFile(metaPath)
bad = true
}
}
- var sha256Digest string
- var sha512Digest string
- fd, err := os.Open(f.Name)
- if err != nil {
- continue
- }
- for _, h := range f.Hashes {
- switch h.Type {
- case meta4ra.HashSHA256:
- sha256Digest = h.Hash
- case meta4ra.HashSHA512:
- sha512Digest = h.Hash
+ hasher := meta4ra.NewHasher(*hashes)
+ var hashTheir string
+ var hashName string
+ for i, name := range hasher.Names {
+ for _, h := range f.Hashes {
+ if h.Type == name {
+ hasher.Names = []string{name}
+ hasher.Cmds = append(hasher.Cmds[:0], hasher.Cmds[i])
+ hasher.Ins = append(hasher.Ins[:0], hasher.Ins[i])
+ hasher.Outs = append(hasher.Outs[:0], hasher.Outs[i])
+ hashName = name
+ hashTheir = h.Hash
+ goto HashFound
+ }
}
}
- var digestTheir string
- var digestName string
- var hasher hash.Hash
- if sha512Digest != "" {
- digestName = meta4ra.HashSHA512
- digestTheir = sha512Digest
- hasher = sha512Hasher
- } else if sha256Digest != "" {
- digestName = meta4ra.HashSHA256
- digestTheir = sha256Digest
- hasher = sha256Hasher
- } else {
- fd.Close()
- fmt.Println("Error:", f.Name, "no satisfiable hash algorithm found")
- bad = true
+ log.Fatalln("no common hashes found for:", f.Name)
+ HashFound:
+ fd, err := os.Open(f.Name)
+ if err != nil {
continue
}
- hasher.Reset()
- _, err = io.Copy(hasher, bufio.NewReader(fd))
+ hasher.Start()
+ _, err = io.Copy(hasher, bufio.NewReaderSize(fd, 1<<20))
fd.Close()
+ sums := hasher.Sums()
if err != nil {
fmt.Println("Error:", f.Name, err)
bad = true
continue
}
- digestOur := hex.EncodeToString(hasher.Sum(nil))
- if digestOur == digestTheir {
- fmt.Println(f.Name, digestName, "good")
+ hashOur := sums[0].Hash
+ if hashOur == hashTheir {
+ fmt.Println(f.Name, hashName, "good")
} else {
fmt.Println(
- "Hash does not match:", f.Name, digestName,
- "our:", digestOur,
- "their:", digestTheir,
+ "Hash does not match:", f.Name, hashName,
+ "our:", hashOur,
+ "their:", hashTheir,
)
bad = true
continue
import (
"bufio"
- "bytes"
"encoding/xml"
"flag"
"io"
"log"
"os"
- "os/exec"
"path"
"strings"
- "sync"
"time"
"go.stargrave.org/meta4ra"
)
-type Hasher struct {
- names []string
- cmds []*exec.Cmd
- ins []io.WriteCloser
- outs []io.ReadCloser
- wg sync.WaitGroup
-}
-
-func NewHasher(hashes string) *Hasher {
- h := Hasher{}
- for _, hc := range strings.Split(hashes, ",") {
- cols := strings.SplitN(hc, ":", 2)
- name, cmdline := cols[0], cols[1]
- cmd := exec.Command(cmdline)
- in, err := cmd.StdinPipe()
- if err != nil {
- log.Fatalln(err)
- }
- out, err := cmd.StdoutPipe()
- if err != nil {
- log.Fatalln(err)
- }
- if err = cmd.Start(); err != nil {
- log.Fatalln(err)
- }
- h.names = append(h.names, name)
- h.ins = append(h.ins, in)
- h.outs = append(h.outs, out)
- h.cmds = append(h.cmds, cmd)
- }
- return &h
-}
-
-func (h *Hasher) Write(p []byte) (n int, err error) {
- h.wg.Add(len(h.names))
- for _, in := range h.ins {
- go func(in io.WriteCloser) {
- if _, err := io.Copy(in, bytes.NewReader(p)); err != nil {
- log.Fatalln(err)
- }
- h.wg.Done()
- }(in)
- }
- h.wg.Wait()
- return len(p), nil
-}
-
-func (h *Hasher) Sums() []meta4ra.Hash {
- sums := make([]meta4ra.Hash, 0, len(h.names))
- for i, name := range h.names {
- if err := h.ins[i].Close(); err != nil {
- log.Fatalln(err)
- }
- dgst, err := io.ReadAll(h.outs[i])
- if err != nil {
- log.Fatalln(err)
- }
- sums = append(sums, meta4ra.Hash{Type: name, Hash: string(dgst[:len(dgst)-1])})
- if err = h.cmds[i].Wait(); err != nil {
- log.Fatalln(err)
- }
- }
- return sums
-}
-
func main() {
fn := flag.String("fn", "", "Filename")
mtime := flag.String("mtime", "", "Take that file's mtime as a Published date")
desc := flag.String("desc", "", "Description")
sig := flag.String("sig", "", "Path to signature file")
- hashesDef := []string{
- "sha-256:sha256",
- "sha-512:sha512",
- "shake128:shake128sum",
- "shake256:shake256sum",
- "streebog-256:streebog256sum",
- "streebog-512:streebog512sum",
- "skein-256:skein256",
- "skein-512:skein512",
- "blake3-256:b3sum",
- }
- hashes := flag.String("hashes", strings.Join(hashesDef, ","), "hash-name:command-s")
+ hashes := flag.String("hashes", strings.Join(meta4ra.HashesDefault, ","), "hash-name:command-s")
torrent := flag.String("torrent", "", "Torrent URL")
log.SetFlags(log.Lshortfile)
flag.Parse()
for _, u := range flag.Args() {
urls = append(urls, meta4ra.URL{URL: u})
}
- h := NewHasher(*hashes)
+ h := meta4ra.NewHasher(*hashes)
+ h.Start()
br := bufio.NewReaderSize(os.Stdin, 1<<20)
buf := make([]byte, 1<<20)
size, err := io.CopyBuffer(h, br, buf)
// Metalink 4.0 utilities
package meta4ra
-import (
- "encoding/xml"
- "time"
-)
-
const (
Generator = "meta4ra/0.3.0"
GPGSigMediaType = "application/pgp-signature"
)
-
-type Metalink struct {
- XMLName xml.Name `xml:"urn:ietf:params:xml:ns:metalink metalink"`
- Files []File `xml:"file"`
- Generator string `xml:"generator,,omitempty"`
- Published time.Time `xml:"published,,omitempty"`
-}
-
-type File struct {
- XMLName xml.Name `xml:"file"`
- Name string `xml:"name,attr"`
- Description string `xml:"description,,omitempty"`
- Hashes []Hash `xml:"hash,,omitempty"`
- MetaURLs []MetaURL `xml:"metaurl,,omitempty"`
- Signature *Signature `xml:"signature"`
- Size uint64 `xml:"size,,omitempty"`
- URLs []URL `xml:"url,,omitempty"`
-}
-
-type URL struct {
- XMLName xml.Name `xml:"url"`
- URL string `xml:",chardata"`
-}
-
-type Signature struct {
- XMLName xml.Name `xml:"signature"`
- MediaType string `xml:"mediatype,attr"`
- Signature string `xml:",cdata"`
-}
-
-type Hash struct {
- XMLName xml.Name `xml:"hash"`
- Type string `xml:"type,attr"`
- Hash string `xml:",chardata"`
-}
-
-type MetaURL struct {
- XMLName xml.Name `xml:"metaurl"`
- MediaType string `xml:"mediatype,attr"`
- URL string `xml:",chardata"`
-}
--- /dev/null
+package meta4ra
+
+import (
+ "bytes"
+ "io"
+ "log"
+ "os/exec"
+ "strings"
+ "sync"
+)
+
+// Sorted by preference order.
+var HashesDefault = []string{
+ "blake3-256:b3sum",
+ "skein-512:skein512",
+ "skein-256:skein256",
+ "shake128:goshake128",
+ "shake256:goshake256",
+ "sha-512:sha512",
+ "sha-256:sha256",
+ "streebog-256:streebog256sum",
+ "streebog-512:streebog512sum",
+}
+
+type Hasher struct {
+ Names []string
+ Cmds []*exec.Cmd
+ Ins []io.WriteCloser
+ Outs []io.ReadCloser
+ wg sync.WaitGroup
+}
+
+func NewHasher(hashes string) *Hasher {
+ h := Hasher{}
+ for _, hc := range strings.Split(hashes, ",") {
+ cols := strings.SplitN(hc, ":", 2)
+ name, cmdline := cols[0], cols[1]
+ cmd := exec.Command(cmdline)
+ in, err := cmd.StdinPipe()
+ if err != nil {
+ log.Fatalln(err)
+ }
+ out, err := cmd.StdoutPipe()
+ if err != nil {
+ log.Fatalln(err)
+ }
+ h.Names = append(h.Names, name)
+ h.Ins = append(h.Ins, in)
+ h.Outs = append(h.Outs, out)
+ h.Cmds = append(h.Cmds, cmd)
+ }
+ return &h
+}
+
+func (h *Hasher) Start() {
+ for _, cmd := range h.Cmds {
+ if err := cmd.Start(); err != nil {
+ log.Fatalln(err)
+ }
+ }
+}
+
+func (h *Hasher) Write(p []byte) (n int, err error) {
+ h.wg.Add(len(h.Names))
+ for _, in := range h.Ins {
+ go func(in io.WriteCloser) {
+ if _, err := io.Copy(in, bytes.NewReader(p)); err != nil {
+ log.Fatalln(err)
+ }
+ h.wg.Done()
+ }(in)
+ }
+ h.wg.Wait()
+ return len(p), nil
+}
+
+func (h *Hasher) Sums() []Hash {
+ sums := make([]Hash, 0, len(h.Names))
+ for i, name := range h.Names {
+ if err := h.Ins[i].Close(); err != nil {
+ log.Fatalln(err)
+ }
+ dgst, err := io.ReadAll(h.Outs[i])
+ if err != nil {
+ log.Fatalln(err)
+ }
+ sums = append(sums, Hash{Type: name, Hash: string(dgst[:len(dgst)-1])})
+ if err = h.Cmds[i].Wait(); err != nil {
+ log.Fatalln(err)
+ }
+ }
+ return sums
+}
--- /dev/null
+/*
+meta4a -- Metalink 4.0 creator
+Copyright (C) 2021-2023 Sergey Matveev <stargrave@stargrave.org>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, version 3 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package meta4ra
+
+import (
+ "encoding/xml"
+ "time"
+)
+
+type Metalink struct {
+ XMLName xml.Name `xml:"urn:ietf:params:xml:ns:metalink metalink"`
+ Files []File `xml:"file"`
+ Generator string `xml:"generator,,omitempty"`
+ Published time.Time `xml:"published,,omitempty"`
+}
+
+type File struct {
+ XMLName xml.Name `xml:"file"`
+ Name string `xml:"name,attr"`
+ Description string `xml:"description,,omitempty"`
+ Hashes []Hash `xml:"hash,,omitempty"`
+ MetaURLs []MetaURL `xml:"metaurl,,omitempty"`
+ Signature *Signature `xml:"signature"`
+ Size uint64 `xml:"size,,omitempty"`
+ URLs []URL `xml:"url,,omitempty"`
+}
+
+type URL struct {
+ XMLName xml.Name `xml:"url"`
+ URL string `xml:",chardata"`
+}
+
+type Signature struct {
+ XMLName xml.Name `xml:"signature"`
+ MediaType string `xml:"mediatype,attr"`
+ Signature string `xml:",cdata"`
+}
+
+type Hash struct {
+ XMLName xml.Name `xml:"hash"`
+ Type string `xml:"type,attr"`
+ Hash string `xml:",chardata"`
+}
+
+type MetaURL struct {
+ XMLName xml.Name `xml:"metaurl"`
+ MediaType string `xml:"mediatype,attr"`
+ URL string `xml:",chardata"`
+}