From 2670391782b1f2a21f004d4603f16b8522d5aaf1bb35d10084e315b6069b415e Mon Sep 17 00:00:00 2001 From: Sergey Matveev Date: Sat, 14 Mar 2026 19:54:19 +0300 Subject: [PATCH] Be less dependent on meta4ra-dl --- build/distfiles/dl/default.rc | 16 ++++++------- build/distfiles/dl/gogost-v6.1.0-modcache.do | 2 +- build/distfiles/dl/gosha3-v1.1.0-modcache.do | 4 ++-- doc/Build/Distfiles/Fetchers | 22 +++++++++-------- doc/Build/Distfiles/Mirrors | 2 ++ doc/Build/Distfiles/Tutorial | 11 +++++++-- doc/Build/Requirements | 15 ++++++------ lib/rc | 25 ++++++++++++++------ 8 files changed, 60 insertions(+), 37 deletions(-) diff --git a/build/distfiles/dl/default.rc b/build/distfiles/dl/default.rc index 38e9e2c..4b4b551 100644 --- a/build/distfiles/dl/default.rc +++ b/build/distfiles/dl/default.rc @@ -32,6 +32,7 @@ if [ -r cache/$dst ] ; then fi ( + # do not depend on generated URLs for v in $(env | sed "s/=.*$//" | grep "^REDO") ; do unset $v ; done redo-ifchange "$metadir"/urls ) @@ -49,13 +50,13 @@ tmp=$(mktemp -d ${TMPDIR:-/tmp}/$fn.XXXXXX) trap "rm -fr $tmp" HUP PIPE INT QUIT TERM EXIT cd $tmp case "$DISTFILE_FETCHER" in -meta4ra) - [ -s "$metadir"/size ] && read size <"$metadir"/size || size=0 +justdl) FETCHER_URLS_SORT <"$metadir"/urls | while read url ; do echo trying ${url}... >&2 + url="${url##*|}" if [ -s "$metadir"/hashes ] ; then - meta4ra-dl -progress -size $size "$url" | + $JUSTDL "$url" | "$BASS_ROOT"/build/bin/hashes-check "$metadir"/hashes "$BASS_HASHERS" \ >$fn || { echo hash check failed >&2 @@ -64,10 +65,9 @@ meta4ra) } else echo warning: no $metadir/hashes >&2 - if [ "$size" -eq 0 ] ; then - meta4ra-dl -progress "$url" >$fn || continue - else - meta4ra-dl -progress -size $size "$url" >$fn || continue + $JUSTDL "$url" >$fn || continue + if [ -s "$metadir"/size ] ; then + read size <"$metadir"/size sizeGot=$("$BASS_ROOT"/bin/filessize $fn) [ "$sizeGot" -eq "$size" ] || { echo size differs: $sizeGot '!=' $size >&2 @@ -80,7 +80,7 @@ meta4ra) [ -s $fn ] cat $fn ;; -wget) +wget-meta4) get-meta4 >meta4 wget --input-metalink=meta4 >&2 cat "$fn.meta4.#1" diff --git a/build/distfiles/dl/gogost-v6.1.0-modcache.do b/build/distfiles/dl/gogost-v6.1.0-modcache.do index c208ae1..0b412d7 100644 --- a/build/distfiles/dl/gogost-v6.1.0-modcache.do +++ b/build/distfiles/dl/gogost-v6.1.0-modcache.do @@ -6,7 +6,7 @@ cd cache/download/go.cypherpunks.su/gogost/v6/@v for ext in info mod zip ziphash ziphash.sig ; do fn=v6.1.0.$ext [ ! -s $fn ] || continue - $JUSTDL tmp http://proxy.go.cypherpunks.su/go.cypherpunks.su/gogost/v6/@v/$fn + $JUSTDL http://proxy.go.cypherpunks.su/go.cypherpunks.su/gogost/v6/@v/$fn >tmp $FSYNC tmp mv tmp $fn done diff --git a/build/distfiles/dl/gosha3-v1.1.0-modcache.do b/build/distfiles/dl/gosha3-v1.1.0-modcache.do index 7fa1b57..81c977f 100644 --- a/build/distfiles/dl/gosha3-v1.1.0-modcache.do +++ b/build/distfiles/dl/gosha3-v1.1.0-modcache.do @@ -6,13 +6,13 @@ vdir="$GOMODCACHE"/cache/download/go.stargrave.org/gosha3/@v mkdir -p $vdir cd $vdir url=http://proxy.go.stargrave.org/go.stargrave.org/gosha3/@v -$JUSTDL tmp $url/v1.1.0.zip +$JUSTDL $url/v1.1.0.zip >tmp $FSYNC tmp [ "$("$BASS_ROOT"/build/lib/zip-hash/zip-hash tmp)" = "h1:9FDmAEU07dVhWV5GYoaetbspzU5u+2amPXClwvHVZ6Q=" ] mv tmp v1.1.0.zip for ext in info mod ziphash ; do fn=v1.1.0.$ext - $JUSTDL tmp $url/$fn + $JUSTDL $url/$fn >tmp $FSYNC tmp mv tmp $fn done diff --git a/doc/Build/Distfiles/Fetchers b/doc/Build/Distfiles/Fetchers index 12f8acc..0af3421 100644 --- a/doc/Build/Distfiles/Fetchers +++ b/doc/Build/Distfiles/Fetchers @@ -11,20 +11,22 @@ $DISTFILES/bin/meta4-to-metadir uses meta4ra utilities to convert .meta4 file to "metadir". $DISTFILES/bin/metadir-to-meta4 generates .meta4 back. -* $DISTFILE_FETCHER=meta4ra - => http://www.meta4ra.stargrave.org/ - Use meta4ra-dl for downloading. URLs are sorted with FETCHER_URLS_SORT - function, which is by default: +A [Index/Variables] JUSTDL +* $DISTFILE_FETCHER=justdl + * Get list or possible download URLs. It is sorted with + FETCHER_URLS_SORT function, which is by default: FETCHER_URLS_SORT() { $DISTFILES/lib/urls-sort "" rand } - This is by default, because meta4ra utilities are installed by default. - -* $DISTFILE_FETCHER=wget + * For each URL, until we succeed, run $JUSTDL program to fetch it. + If JUSTDL is not set, then BASS will check for following programs + existence: fetch, wget, curl, meta4ra-dl. + * Check downloaded result with $BASS_ROOT/build/bin/hashes-check + utility against $metadir/hashes, if it exists. Otherwise check + only the size. +* $DISTFILE_FETCHER=wget-meta4 => https://www.gnu.org/software/wget/ GNU Wget - Use wget compiled with --with-metalink option. The only drawback is - that most OS distributions contain Wget without that (--input-metalink) - option. + Use wget compiled with --with-metalink option. * $DISTFILE_FETCHER=aria2c => http://aria2.github.io/ Aria2 diff --git a/doc/Build/Distfiles/Mirrors b/doc/Build/Distfiles/Mirrors index ed50235..0993684 100644 --- a/doc/Build/Distfiles/Mirrors +++ b/doc/Build/Distfiles/Mirrors @@ -9,6 +9,8 @@ Here we use the following precedence preference: * all links should use HTTP as higher priority than HTTPS * distcache.FreeBSD.org and cdn.NetBSD.org/pub/pkgsrc/distfiles are used as least priority fallback +* be aware that codeload.github.com downloads have no guarantees + of format/compression stability (their hashes may change) There are special country codes for identifying and specifying CDNs: xa -> Akamai diff --git a/doc/Build/Distfiles/Tutorial b/doc/Build/Distfiles/Tutorial index 163f41d..b65dad8 100644 --- a/doc/Build/Distfiles/Tutorial +++ b/doc/Build/Distfiles/Tutorial @@ -33,6 +33,12 @@ will use hashes-gen script: $ $BASS_ROOT/build/bin/hashes-gen <$tarball >$DISTFILES/meta/$tarball/hashes +Or we can use helper script doing completely the same: + + $ cd $DISTFILES/meta + $ ../bin/metadir-from-file /path/to/$tarball + $ ../bin/metadir-from-file /path/to/$tarball.sig + Then we must store a list of URLs where it can be fetched. That may be just trivial: @@ -48,9 +54,10 @@ Fortunately there is one for GNU ecosystem. You should automatically generate your urls file instead then. $ cat >$DISTFILES/meta/$tarball/urls.do < http://www.meta4ra.stargrave.org/ meta4ra - Utilities for making and checking .meta4 files. They are just a - wrapper over XML and external hasher commands interoperation. They - also can be used for downloading. - A [Index/Programs] Perl => https://www.perl.org/ Perl Shell scripts are hard to write in a portable way. For example there @@ -57,10 +51,17 @@ at least: build Go-related software. Actually Go-written utilities can be replaced and no Go dependency will be required at all. + A [Index/Programs] meta4ra +=> http://www.meta4ra.stargrave.org/ meta4ra + Optional utilities for making and checking .meta4 files. + They are just a wrapper over XML and external hasher + commands interoperation. They also can be used for downloading. + A [Index/Programs] fetch A [Index/Programs] Wget A [Index/Programs] cURL * FreeBSD's fetch, or => https://www.gnu.org/software/wget/ GNU Wget => https://curl.se/ cURL - Although meta4ra can be used instead all of them. + => http://www.meta4ra.stargrave.org/ meta4ra-dl + To fetch distfiles. diff --git a/lib/rc b/lib/rc index 20c114d..b95a7ed 100644 --- a/lib/rc +++ b/lib/rc @@ -33,11 +33,22 @@ COMPRESSOR=${COMPRESSOR:-zstdmt} export TMPDIR=${TMPDIR:-/tmp} MAKE_JOBS=${MAKE_JOBS:-$(nproc)} -if command -v fetch >/dev/null 2>/dev/null ; then - JUSTDL="fetch -o" -else - command -v wget >/dev/null 2>/dev/null && JUSTDL="wget -O" || JUSTDL="curl -o" -fi +[ -n "$JUSTDL" ] || { + command -v fetch >/dev/null 2>/dev/null && JUSTDL="fetch -v -o -" +} +[ -n "$JUSTDL" ] || { + command -v wget >/dev/null 2>/dev/null && JUSTDL="wget -O -" +} +[ -n "$JUSTDL" ] || { + command -v curl >/dev/null 2>/dev/null && JUSTDL="curl -f -v" +} +[ -n "$JUSTDL" ] || { + command -v meta4ra-dl >/dev/null 2>/dev/null && JUSTDL="meta4ra-dl" +} +[ -n "$JUSTDL" ] || { + echo unable to find suitable \$JUSTDL utility >&2 + exit 1 +} export DISTFILES=${DISTFILES:-"$BASS_ROOT"/build/distfiles} export SKELBINS=${SKELBINS:-/tmp/skelbins} @@ -46,8 +57,8 @@ export SKELPKGS=${SKELPKGS:-"$BASS_ROOT"/build/pkg} [ -n "$PV" ] || command -v pv >/dev/null 2>/dev/null && PV=pv || PV=cat -# Program to use for downloading distfiles: "meta4ra", "wget", "aria2c". -export DISTFILE_FETCHER="${DISTFILE_FETCHER:-meta4ra}" +# Program to use for downloading distfiles: "justdl", "wget-meta4", "aria2c". +export DISTFILE_FETCHER="${DISTFILE_FETCHER:-justdl}" if ! type FETCHER_URLS_SORT 2>/dev/null >/dev/null ; then FETCHER_URLS_SORT() { $DISTFILES/lib/urls-sort "" rand -- 2.52.0