From: Sergey Matveev Date: Mon, 16 Feb 2026 20:41:06 +0000 (+0300) Subject: meta4ra-dl/list/url-sort X-Git-Tag: v1.0.0~2 X-Git-Url: http://www.git.stargrave.org/?a=commitdiff_plain;h=8b627058f51e39a9cbce6f9746496d152ed54191;p=meta4ra.git meta4ra-dl/list/url-sort --- diff --git a/README b/README index 0f8a2f0..a3fec86 100644 --- a/README +++ b/README @@ -4,13 +4,38 @@ meta4ra-create utility is used to create Metalink4 (https://datatracker.ietf.org/doc/html/rfc5854) .meta4-file for single specified file. -meta4ra-check utility is used to check .meta4 file, extract signatures -and verify corresponding files integrity. Optionally download files. +meta4ra-list utility is used to list files in .meta4 file, get its size, +list its URLs in "pri|cc|URL" format, extract signatures. + +meta4ra-check utility is used verify corresponding files integrity. + +meta4ra-url-sort utility is used to sort "pri|cc|URL" lines by specified +country codes, continents, priority. + +meta4ra-dl utility can be used to download specified URL. meta4ra-hash utility can be used to hash the data with a single hash algorithm. It could be useful if you want to use the best found algorithm on current system, whatever it is. +Example attempt to download .meta4's file from Russia-based location as +most preferred, with fallback to Europe continent, leaving location-less +next, randomising remaining: + + fn=$(meta4ra-list -meta4 .meta4 | head -1) + size=$(meta4ra-list -meta4 .meta4 -size $fn) + meta4ra-list -meta4 .meta4 $fn | + meta4ra-url-sort ru c:eu "" rand | + while read url ; do + meta4ra-dl -size $size -progress "$url" | + meta4ra-check -meta4 .meta4 -pipe $fn >$fn || { + rm $fn + continue + } + break + done + [ -s $fn ] + meta4ra is copylefted free software: see the file COPYING for copying conditions. It should work on all POSIX-compatible systems. diff --git a/bin/meta4ra-dl b/bin/meta4ra-dl new file mode 120000 index 0000000..ebcf0ba --- /dev/null +++ b/bin/meta4ra-dl @@ -0,0 +1 @@ +meta4ra \ No newline at end of file diff --git a/bin/meta4ra-list b/bin/meta4ra-list new file mode 120000 index 0000000..ebcf0ba --- /dev/null +++ b/bin/meta4ra-list @@ -0,0 +1 @@ +meta4ra \ No newline at end of file diff --git a/bin/meta4ra-url-sort b/bin/meta4ra-url-sort new file mode 120000 index 0000000..ebcf0ba --- /dev/null +++ b/bin/meta4ra-url-sort @@ -0,0 +1 @@ +meta4ra \ No newline at end of file diff --git a/cmd/meta4ra/check.go b/cmd/meta4ra/check.go index 485f2cd..560936c 100644 --- a/cmd/meta4ra/check.go +++ b/cmd/meta4ra/check.go @@ -21,12 +21,9 @@ import ( "flag" "fmt" "io" - "io/fs" "log" - "net/http" "os" "path" - "strings" "time" meta4ra "go.stargrave.org/meta4ra/internal" @@ -38,9 +35,7 @@ func runCheck() { allHashes := flag.Bool("all-hashes", false, "Check all hashes, not the first common one") hashes := flag.String("hashes", meta4ra.HashesDefault, "hash-name:commandline[,...]") - extractSig := flag.Bool("extract-sig", false, "Extract signature files") metaPath := flag.String("meta4", "file.meta4", "Metalink file") - dl := flag.Int("dl", -1, "URL index to download, instead of reading from stdin") flag.Usage = func() { fmt.Fprintf(flag.CommandLine.Output(), "Usage: %s [options] [FILE ...]\n", os.Args[0]) @@ -48,12 +43,8 @@ func runCheck() { fmt.Fprint(flag.CommandLine.Output(), ` If no FILEs are specified, then all s from metalink are searched and verified if they exist. Otherwise only specified FILEs are checked. If you -want to skip any verification (for example only to validate the format -and -extract-sig, then you can just specify an empty ("") FILE. - -If -dl (> 0) is specified, then it automatically implies -pipe, but -downloads data by specified URLs index, instead of reading from stdin. -That can be used as a downloading utility. +want to skip any verification (for example only to validate the +format, then you can just specify an empty ("") FILE. `) } flag.Parse() @@ -67,53 +58,28 @@ That can be used as a downloading utility. return } - data, err := os.ReadFile(*metaPath) - if err != nil { - log.Fatalln(err) - } var meta meta4ra.Metalink - err = xml.Unmarshal(data, &meta) - if err != nil { - log.Fatalln(err) + { + data, err := os.ReadFile(*metaPath) + if err != nil { + log.Fatal(err) + } + err = xml.Unmarshal(data, &meta) + if err != nil { + log.Fatal(err) + } } toCheck := make(map[string]string) for _, fn := range flag.Args() { toCheck[path.Base(fn)] = fn } - if *dl != -1 { - *pipe = true - } if *pipe && len(toCheck) != 1 { - log.Fatalln("exactly single FILE must be specified when using -pipe") + log.Fatal("exactly single FILE must be specified when using -pipe") } bad := false for _, f := range meta.Files { - for _, sig := range f.Signature { - if !*extractSig { - continue - } - var fn string - switch sig.MediaType { - case meta4ra.SigMediaTypePGP: - fn = f.Name + ".asc" - case meta4ra.SigMediaTypeSSH: - fn = f.Name + ".sig" - } - if fn == "" { - continue - } - if err = os.WriteFile( - fn, - []byte(strings.TrimPrefix(sig.Signature, "\n")), - fs.FileMode(0o666), - ); err != nil { - log.Println("Error:", f.Name, "can not save signature:", err) - bad = true - } - } - fullPath := toCheck[f.Name] delete(toCheck, f.Name) if !(len(toCheck) == 0 || fullPath != "") { @@ -176,25 +142,6 @@ That can be used as a downloading utility. continue } } - if *dl != -1 { - var resp *http.Response - resp, err = http.Get(f.URLs[*dl].URL) - if err != nil { - log.Println("Error:", f.Name, err) - bad = true - continue - } - log.Println("HTTP response:") - for k := range resp.Header { - log.Printf("\t%+q: %+q\n", k, resp.Header.Get(k)) - } - if resp.StatusCode != http.StatusOK { - log.Println("Bad status code:", f.Name, resp.Status) - bad = true - continue - } - src = resp.Body - } err = hasher.Start() if err != nil { if !*pipe { @@ -217,7 +164,7 @@ That can be used as a downloading utility. w = &bar } _, err = io.Copy(w, bufio.NewReaderSize(src, meta4ra.BufLen)) - if !*pipe || *dl != -1 { + if !*pipe { src.Close() } if err != nil { diff --git a/cmd/meta4ra/create.go b/cmd/meta4ra/create.go index 37a2ebb..18831b1 100644 --- a/cmd/meta4ra/create.go +++ b/cmd/meta4ra/create.go @@ -49,6 +49,10 @@ func runCreate() { fmt.Fprintf(flag.CommandLine.Output(), "Usage: %s [options] [URL ...] data.meta4\n", os.Args[0]) flag.PrintDefaults() + fmt.Fprint(flag.CommandLine.Output(), ` +If URL is specified in "p|cc|URL" format, then "p" (may be empty) +will be the priority and "cc" (may be empty) is location. +`) } flag.Parse() @@ -62,26 +66,30 @@ func runCreate() { } if *fn == "" { - log.Fatalln("empty -fn") + log.Fatal("empty -fn") } urls := make([]meta4ra.URL, 0, len(flag.Args())) for _, u := range flag.Args() { - urls = append(urls, meta4ra.URL{URL: u}) + url, err := meta4ra.ParseURL(u) + if err != nil { + log.Fatal(err) + } + urls = append(urls, *url) } h, err := meta4ra.NewHasher(*hashes) if err != nil { - log.Fatalln(err) + log.Fatal(err) } h.Start() br := bufio.NewReaderSize(os.Stdin, meta4ra.BufLen) buf := make([]byte, meta4ra.BufLen) size, err := io.CopyBuffer(h, br, buf) if err != nil { - log.Fatalln(err) + log.Fatal(err) } dgsts, err := h.Sums() if err != nil { - log.Fatalln(err) + log.Fatal(err) } f := meta4ra.File{ Name: path.Base(*fn), @@ -94,7 +102,7 @@ func runCreate() { var sigData []byte sigData, err = os.ReadFile(*sigPGP) if err != nil { - log.Fatalln(err) + log.Fatal(err) } f.Signature = append(f.Signature, meta4ra.Signature{ MediaType: meta4ra.SigMediaTypePGP, @@ -105,7 +113,7 @@ func runCreate() { var sigData []byte sigData, err = os.ReadFile(*sigSSH) if err != nil { - log.Fatalln(err) + log.Fatal(err) } f.Signature = append(f.Signature, meta4ra.Signature{ MediaType: meta4ra.SigMediaTypeSSH, @@ -122,7 +130,7 @@ func runCreate() { var fi fs.FileInfo fi, err = os.Stat(*mtime) if err != nil { - log.Fatalln(err) + log.Fatal(err) } published = fi.ModTime() } @@ -136,7 +144,7 @@ func runCreate() { } out, err := xml.MarshalIndent(&m, "", " ") if err != nil { - log.Fatalln(err) + log.Fatal(err) } os.Stdout.Write([]byte(xml.Header)) os.Stdout.Write(out) diff --git a/cmd/meta4ra/dl.go b/cmd/meta4ra/dl.go new file mode 100644 index 0000000..f8dc54e --- /dev/null +++ b/cmd/meta4ra/dl.go @@ -0,0 +1,78 @@ +// meta4ra -- Metalink 4.0 utilities +// Copyright (C) 2021-2026 Sergey Matveev +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, version 3 of the License. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +package main + +import ( + "bufio" + "flag" + "fmt" + "io" + "log" + "net/http" + "os" + "time" + + meta4ra "go.stargrave.org/meta4ra/internal" +) + +func runDl() { + progress := flag.Bool("progress", false, "Show progress of downloading") + size := flag.Uint("size", 0, "Set expected size for -progress") + flag.Usage = func() { + fmt.Fprintf(flag.CommandLine.Output(), + "Usage: %s [options] URL >data\n", os.Args[0]) + flag.PrintDefaults() + } + flag.Parse() + + if *showVersion { + fmt.Println(meta4ra.Version()) + return + } + if *showWarranty { + fmt.Println(meta4ra.Warranty) + return + } + + u, err := meta4ra.ParseURL(flag.Arg(0)) + if err != nil { + log.Fatal(err) + } + resp, err := http.Get(u.URL) + if err != nil { + log.Fatal(err) + } + log.Println("HTTP response:") + for k := range resp.Header { + log.Printf("\t%+q: %+q\n", k, resp.Header.Get(k)) + } + if resp.StatusCode != http.StatusOK { + log.Fatalln("status code:", resp.Status) + } + var w io.Writer + if *progress { + bar := Progress{w: os.Stdout, now: time.Now(), total: uint64(*size)} + bar.next = bar.now.Add(ProgressPeriod) + w = &bar + } else { + w = os.Stdout + } + _, err = io.Copy(w, bufio.NewReaderSize(resp.Body, meta4ra.BufLen)) + resp.Body.Close() + if err != nil { + log.Fatal(err) + } +} diff --git a/cmd/meta4ra/hash.go b/cmd/meta4ra/hash.go index 6fdce94..579aea3 100644 --- a/cmd/meta4ra/hash.go +++ b/cmd/meta4ra/hash.go @@ -55,18 +55,18 @@ Only the first hash from -hashes will be used. } hasher, err := meta4ra.NewHasher(hsh) if err != nil { - log.Fatalln(err) + log.Fatal(err) } if err = hasher.Start(); err != nil { - log.Fatalln(err) + log.Fatal(err) } if _, err = io.Copy(hasher, bufio.NewReaderSize( os.Stdin, meta4ra.BufLen)); err != nil { - log.Fatalln(err) + log.Fatal(err) } sums, err := hasher.Sums() if err != nil { - log.Fatalln(err) + log.Fatal(err) } fmt.Println(sums[0].Hash, sums[0].Type) } diff --git a/cmd/meta4ra/list.go b/cmd/meta4ra/list.go new file mode 100644 index 0000000..73757e8 --- /dev/null +++ b/cmd/meta4ra/list.go @@ -0,0 +1,110 @@ +// meta4ra -- Metalink 4.0 utilities +// Copyright (C) 2021-2026 Sergey Matveev +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, version 3 of the License. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +package main + +import ( + "encoding/xml" + "flag" + "fmt" + "io/fs" + "log" + "os" + "strings" + + meta4ra "go.stargrave.org/meta4ra/internal" +) + +func runList() { + metaPath := flag.String("meta4", "file.meta4", "Metalink file") + doSize := flag.Bool("size", false, "Print file's size") + doSig := flag.Bool("sig", false, "Extract signature files") + flag.Usage = func() { + fmt.Fprintf(flag.CommandLine.Output(), + "Usage: %s [options] [FILE]\n", os.Args[0]) + flag.PrintDefaults() + fmt.Fprint(flag.CommandLine.Output(), ` +If FILE is not specified, then list all files of .meta4. +Otherwise list all URLs of the given FILE. +`) + } + flag.Parse() + + if *showVersion { + fmt.Println(meta4ra.Version()) + return + } + if *showWarranty { + fmt.Println(meta4ra.Warranty) + return + } + + var meta meta4ra.Metalink + { + data, err := os.ReadFile(*metaPath) + if err != nil { + log.Fatal(err) + } + err = xml.Unmarshal(data, &meta) + if err != nil { + log.Fatal(err) + } + } + + bad := false + for _, f := range meta.Files { + if flag.NArg() == 0 { + for _, sig := range f.Signature { + if !*doSig { + continue + } + var fn string + switch sig.MediaType { + case meta4ra.SigMediaTypePGP: + fn = f.Name + ".asc" + case meta4ra.SigMediaTypeSSH: + fn = f.Name + ".sig" + } + if fn == "" { + continue + } + if err := os.WriteFile( + fn, + []byte(strings.TrimPrefix(sig.Signature, "\n")), + fs.FileMode(0o666), + ); err != nil { + log.Println("Error:", f.Name, "can not save signature:", err) + bad = true + } + } + fmt.Println(f.Name) + } else { + if f.Name != flag.Arg(0) { + continue + } + if *doSize { + fmt.Println(f.Size) + break + } + for _, u := range f.URLs { + fmt.Printf("%d|%s|%s\n", u.Priority, u.Location, u.URL) + } + break + } + } + if bad { + os.Exit(1) + } +} diff --git a/cmd/meta4ra/main.go b/cmd/meta4ra/main.go index b0a5958..9224025 100644 --- a/cmd/meta4ra/main.go +++ b/cmd/meta4ra/main.go @@ -19,9 +19,15 @@ func main() { runCheck() case "meta4ra-create": runCreate() + case "meta4ra-dl": + runDl() case "meta4ra-hash": runHash() + case "meta4ra-list": + runList() + case "meta4ra-url-sort": + runURLSort() default: - log.Fatalln("unknown command linked") + log.Fatal("unknown command linked") } } diff --git a/cmd/meta4ra/sort.go b/cmd/meta4ra/sort.go new file mode 100644 index 0000000..b36f2ba --- /dev/null +++ b/cmd/meta4ra/sort.go @@ -0,0 +1,143 @@ +// meta4ra -- Metalink 4.0 utilities +// Copyright (C) 2021-2026 Sergey Matveev +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, version 3 of the License. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +package main + +import ( + "bufio" + "flag" + "fmt" + "log" + "math/rand/v2" + "os" + "sort" + "strings" + + meta4ra "go.stargrave.org/meta4ra/internal" +) + +type ByPriority []*meta4ra.URL + +func (urls ByPriority) Len() int { + return len(urls) +} + +func (urls ByPriority) Less(i, j int) bool { + return urls[i].Priority < urls[j].Priority +} + +func (urls ByPriority) Swap(i, j int) { + urls[i], urls[j] = urls[j], urls[i] +} + +var Continents = map[string]string{ + "af": "ao bf bi bj bw cd cf cg ci cm cv dj dz eg eh er et ga gh gm gn gq gw ke km lr ls ly ma mg ml mr mu mw mz na ne ng re rw sc sd sh sl sn so ss st sz td tg tn tz ug yt za zm zw", + "an": "aq bv gs hm tf", + "as": "ae af am as as as az bd bh bn bt cc cn cx cy ge hk id il in io iq ir jo jp kg kh kp kr kw kz la lb lk mm mn mo mv my np om ph pk ps qa ru sa sg sy th tj tl tm tr tw uz vn ye", + "eu": "ad al am at ax az ba be bg by ch cy cz de dk ee es eu fi fo fr ge gg gi gr hr hu ie im is it je kz li lt lu lv mc md me mk mt nl no pl pt ro rs ru se si sj sk sm tr ua uk va", + "na": "ag ai an aw bb bl bm bq bs bz ca cr cu cw dm do gd gl gp gt hn ht jm kn ky lc mf mq ms mx ni pa pm pr sv sx tc tt um us vc vg vi", + "oc": "as au ck fj fm gu ki mh mp nc nf nr nu nz oc pf pg pn pw sb tk to tv um vu wf ws", + "sa": "ar bo br cl co ec fk gf gy pe py sr uy ve", +} + +type ByCC struct { + cc string + urls []*meta4ra.URL +} + +func (by ByCC) Len() int { + return len(by.urls) +} + +func (by ByCC) Less(i, j int) bool { + return strings.Contains(by.cc, by.urls[i].Location) +} + +func (by ByCC) Swap(i, j int) { + by.urls[i], by.urls[j] = by.urls[j], by.urls[i] +} + +func runURLSort() { + flag.Usage = func() { + fmt.Fprintf(flag.CommandLine.Output(), + "Usage: %s [options] [cc ...] = 0; i-- { + cc := flag.Args()[i] + if cc == "rand" { + rand.Shuffle(len(urls), func(i, j int) { + urls[i], urls[j] = urls[j], urls[i] + }) + } + if strings.HasPrefix(cc, "c:") { + cc = Continents[cc[2:]] + } + sort.Stable(ByCC{ + cc: cc, + urls: urls, + }) + } + sort.Stable(ByPriority(urls)) + for _, u := range urls { + if u.Priority == 999999 { + u.Priority = 0 + } + fmt.Println(u) + } +} diff --git a/internal/common.go b/internal/common.go index 2c3ea31..1c19d50 100644 --- a/internal/common.go +++ b/internal/common.go @@ -21,7 +21,7 @@ import ( ) const ( - Generator = "meta4ra/0.12.0" + Generator = "meta4ra/1.0.0" SigMediaTypePGP = "application/pgp-signature" SigMediaTypeSSH = "application/ssh-signature" BufLen = 1 << 20 diff --git a/internal/scheme.go b/internal/scheme.go index f7bee42..e8214c7 100644 --- a/internal/scheme.go +++ b/internal/scheme.go @@ -39,8 +39,10 @@ type File struct { } type URL struct { - XMLName xml.Name `xml:"url"` - URL string `xml:",chardata"` + XMLName xml.Name `xml:"url"` + Priority uint `xml:"priority,attr,omitempty"` + Location string `xml:"location,attr,omitempty"` + URL string `xml:",chardata"` } type Signature struct { diff --git a/internal/url.go b/internal/url.go new file mode 100644 index 0000000..51bf2aa --- /dev/null +++ b/internal/url.go @@ -0,0 +1,47 @@ +// meta4ra -- Metalink 4.0 utilities +// Copyright (C) 2021-2026 Sergey Matveev +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, version 3 of the License. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +package internal + +import ( + "errors" + "fmt" + "strconv" + "strings" +) + +func (u *URL) String() string { + return fmt.Sprintf("%d|%s|%s", u.Priority, u.Location, u.URL) +} + +func ParseURL(s string) (*URL, error) { + cols := strings.SplitN(s, "|", 3) + switch len(cols) { + case 1: + return &URL{URL: cols[0]}, nil + case 3: + p, err := strconv.ParseUint(cols[0], 10, 32) + if err != nil { + return nil, err + } + return &URL{ + Priority: uint(p), + Location: cols[1], + URL: cols[2], + }, nil + default: + return nil, errors.New("bad URL format") + } +} diff --git a/makedist b/makedist index e88ea2c..a19073d 100755 --- a/makedist +++ b/makedist @@ -24,9 +24,9 @@ detpax meta4ra-"$release" >meta4ra-"$release".tar zstd -22 --ultra -v meta4ra-"$release".tar tarball=meta4ra-"$release".tar.zst ssh-keygen -Y sign -f ~/.ssh/sign/meta4ra@stargrave.org -n file $tarball -meta4ra-create -fn "$tarball" -mtime "$tarball" \ - -sig-ssh "$tarball".sig \ - http://www.meta4ra.stargrave.org/download/"$tarball" \ - http://y.www.meta4ra.stargrave.org/download/"$tarball" <"$tarball" >"$tarball".meta4 +meta4ra-create -fn "$tarball" -mtime "$tarball" -sig-ssh "$tarball".sig \ + "1|ru|http://www.meta4ra.stargrave.org/download/$tarball" \ + "2|ru|https://www.meta4ra.stargrave.org/download/$tarball" \ + "http://y.www.meta4ra.stargrave.org/download/$tarball" <"$tarball" >"$tarball".meta4 mv $tmp/$tarball $tarball.meta4 $cur/www/download