From ab67fcb95acbfab9778b6661ce10b2b2bb1bb0aa Mon Sep 17 00:00:00 2001 From: Sergey Matveev Date: Mon, 20 Mar 2023 13:43:02 +0300 Subject: [PATCH] No redo, refactor project building, simplify it --- .gitignore | 5 +- VERSION.do | 2 + all.do | 2 - build | 9 +++ cert.pem.do | 3 - cmd/certgen/.gitignore | 1 + cmd/tofuproxy/.gitignore | 1 + cmd/warc-extract/.gitignore | 1 + cmd/zstd/.gitignore | 1 - cmd/zstd/all.do | 1 - cmd/zstd/build | 4 ++ cmd/zstd/compile_flags.txt.do | 5 -- cmd/zstd/conf/.gitignore | 1 - cmd/zstd/conf/pkgconf.rc.do | 1 - cmd/zstd/conf/zstd.cflags.rc.do | 3 - cmd/zstd/conf/zstd.libs.rc.do | 3 - cmd/zstd/default.do | 5 -- restricted.txt => contrib/restricted.txt | 0 spies.txt => contrib/spies.txt | 0 default.cmd.do | 12 ---- doc/.gitignore | 1 + doc/download.texi | 4 ++ doc/index.texi | 1 + doc/install.texi | 26 ++++++++ doc/restricted.texi | 2 +- doc/spies.texi | 2 +- doc/tofuproxy.info.do | 7 ++ doc/usage.texi | 27 +++----- doc/warcs.texi | 11 ++-- doc/www.do | 3 +- fifos/{ensure.do => ensure} | 2 + fifos/{multitail.sh => multitail} | 2 +- makedist | 82 ++++++++++++++++++++++++ state-init | 16 +++++ version.go | 3 + 35 files changed, 180 insertions(+), 69 deletions(-) create mode 100644 VERSION.do delete mode 100644 all.do create mode 100755 build delete mode 100644 cert.pem.do create mode 100644 cmd/certgen/.gitignore create mode 100644 cmd/tofuproxy/.gitignore create mode 100644 cmd/warc-extract/.gitignore delete mode 100644 cmd/zstd/all.do create mode 100755 cmd/zstd/build delete mode 100644 cmd/zstd/compile_flags.txt.do delete mode 100644 cmd/zstd/conf/.gitignore delete mode 100644 cmd/zstd/conf/pkgconf.rc.do delete mode 100644 cmd/zstd/conf/zstd.cflags.rc.do delete mode 100644 cmd/zstd/conf/zstd.libs.rc.do delete mode 100644 cmd/zstd/default.do rename restricted.txt => contrib/restricted.txt (100%) rename spies.txt => contrib/spies.txt (100%) delete mode 100644 default.cmd.do create mode 100644 doc/.gitignore create mode 100644 doc/download.texi create mode 100644 doc/install.texi create mode 100644 doc/tofuproxy.info.do rename fifos/{ensure.do => ensure} (96%) mode change 100644 => 100755 rename fifos/{multitail.sh => multitail} (98%) create mode 100755 makedist create mode 100755 state-init create mode 100644 version.go diff --git a/.gitignore b/.gitignore index 7af3e1b..5a9f50b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1 @@ -/*.cmd -/ccerts -/cert.pem -/certs +/VERSION diff --git a/VERSION.do b/VERSION.do new file mode 100644 index 0000000..90ff3e2 --- /dev/null +++ b/VERSION.do @@ -0,0 +1,2 @@ +redo-ifchange version.go +perl -ne 'print "$1\n" if /Version.*"(.*)"$/' < version.go diff --git a/all.do b/all.do deleted file mode 100644 index 4f7b66f..0000000 --- a/all.do +++ /dev/null @@ -1,2 +0,0 @@ -redo-ifchange cert.pem tofuproxy.cmd fifos/ensure cmd/zstd/unzstd -mkdir -p certs ccerts diff --git a/build b/build new file mode 100755 index 0000000..d895a71 --- /dev/null +++ b/build @@ -0,0 +1,9 @@ +#!/bin/sh -e + +GO_FLAGS="${GO_FLAGS:--ldflags=-s}" +GO_FLAGS="$GO_FLAGS -mod=vendor" +${GO:-go} build -o cmd/certgen/certgen $GO_FLAGS cmd/certgen/main.go +${GO:-go} build -o cmd/tofuproxy/tofuproxy $GO_FLAGS cmd/tofuproxy/main.go +${GO:-go} build -o cmd/warc-extract/warc-extract $GO_FLAGS cmd/warc-extract/main.go +cmd/zstd/build cmd/zstd/enzstd.c +cmd/zstd/build cmd/zstd/unzstd.c diff --git a/cert.pem.do b/cert.pem.do deleted file mode 100644 index a25ff36..0000000 --- a/cert.pem.do +++ /dev/null @@ -1,3 +0,0 @@ -[ -e certgen.cmd ] || redo certgen.cmd -umask 077 -./certgen.cmd > $3 diff --git a/cmd/certgen/.gitignore b/cmd/certgen/.gitignore new file mode 100644 index 0000000..ab15d20 --- /dev/null +++ b/cmd/certgen/.gitignore @@ -0,0 +1 @@ +/certgen diff --git a/cmd/tofuproxy/.gitignore b/cmd/tofuproxy/.gitignore new file mode 100644 index 0000000..481cc2f --- /dev/null +++ b/cmd/tofuproxy/.gitignore @@ -0,0 +1 @@ +/tofuproxy diff --git a/cmd/warc-extract/.gitignore b/cmd/warc-extract/.gitignore new file mode 100644 index 0000000..19058ea --- /dev/null +++ b/cmd/warc-extract/.gitignore @@ -0,0 +1 @@ +/warc-extract diff --git a/cmd/zstd/.gitignore b/cmd/zstd/.gitignore index 06ed646..f2f013f 100644 --- a/cmd/zstd/.gitignore +++ b/cmd/zstd/.gitignore @@ -1,3 +1,2 @@ -/compile_flags.txt /enzstd /unzstd diff --git a/cmd/zstd/all.do b/cmd/zstd/all.do deleted file mode 100644 index aac1e34..0000000 --- a/cmd/zstd/all.do +++ /dev/null @@ -1 +0,0 @@ -redo-ifchange enzstd unzstd diff --git a/cmd/zstd/build b/cmd/zstd/build new file mode 100755 index 0000000..4ea349b --- /dev/null +++ b/cmd/zstd/build @@ -0,0 +1,4 @@ +#!/bin/sh -e + +PKGCONF="${PKGCONF:-`command -v pkgconf || command -v pkg-config`}" +${CC:-cc} -fstrict-aliasing $CFLAGS `$PKGCONF --cflags libzstd` -o ${1%%.c} $1 `$PKGCONF --libs libzstd` -lpthread diff --git a/cmd/zstd/compile_flags.txt.do b/cmd/zstd/compile_flags.txt.do deleted file mode 100644 index d07460a..0000000 --- a/cmd/zstd/compile_flags.txt.do +++ /dev/null @@ -1,5 +0,0 @@ -redo-ifchange $src conf/zstd.cflags.rc conf/zstd.libs.rc -read ZSTD_CFLAGS < conf/zstd.cflags.rc -read ZSTD_LIBS < conf/zstd.libs.rc -printf "%s" "$CFLAGS $ZSTD_CFLAGS $LDFLAGS $ZSTD_LIBS" | - tr " " "\n" | grep -v "^$" | sort | uniq diff --git a/cmd/zstd/conf/.gitignore b/cmd/zstd/conf/.gitignore deleted file mode 100644 index 27f2869..0000000 --- a/cmd/zstd/conf/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/*.rc diff --git a/cmd/zstd/conf/pkgconf.rc.do b/cmd/zstd/conf/pkgconf.rc.do deleted file mode 100644 index 5b57e6e..0000000 --- a/cmd/zstd/conf/pkgconf.rc.do +++ /dev/null @@ -1 +0,0 @@ -echo ${PKGCONF:-`command -v pkgconf || command -v pkg-config`} diff --git a/cmd/zstd/conf/zstd.cflags.rc.do b/cmd/zstd/conf/zstd.cflags.rc.do deleted file mode 100644 index 2dcf2b2..0000000 --- a/cmd/zstd/conf/zstd.cflags.rc.do +++ /dev/null @@ -1,3 +0,0 @@ -redo-ifchange pkgconf.rc -read PKGCONF < pkgconf.rc -$PKGCONF --cflags libzstd diff --git a/cmd/zstd/conf/zstd.libs.rc.do b/cmd/zstd/conf/zstd.libs.rc.do deleted file mode 100644 index d9b1fb7..0000000 --- a/cmd/zstd/conf/zstd.libs.rc.do +++ /dev/null @@ -1,3 +0,0 @@ -redo-ifchange pkgconf.rc -read PKGCONF < pkgconf.rc -$PKGCONF --libs libzstd diff --git a/cmd/zstd/default.do b/cmd/zstd/default.do deleted file mode 100644 index 56103e2..0000000 --- a/cmd/zstd/default.do +++ /dev/null @@ -1,5 +0,0 @@ -src=$1.c -redo-ifchange $src conf/zstd.cflags.rc conf/zstd.libs.rc -read ZSTD_CFLAGS < conf/zstd.cflags.rc -read ZSTD_LIBS < conf/zstd.libs.rc -${CC:-cc} -fstrict-aliasing $CFLAGS $ZSTD_CFLAGS -static -o $3 $src $ZSTD_LIBS -lpthread diff --git a/restricted.txt b/contrib/restricted.txt similarity index 100% rename from restricted.txt rename to contrib/restricted.txt diff --git a/spies.txt b/contrib/spies.txt similarity index 100% rename from spies.txt rename to contrib/spies.txt diff --git a/default.cmd.do b/default.cmd.do deleted file mode 100644 index 97e1502..0000000 --- a/default.cmd.do +++ /dev/null @@ -1,12 +0,0 @@ -redo-ifchange \ - *.go \ - caches/*.go \ - cmd/$2/*.go \ - fifos/*.go \ - rounds/*.go \ - tls/*.go \ - warc/*.go -UNZSTD_PATH="`pwd`/cmd/zstd/unzstd" -GO_LDFLAGS="${GO_LDFLAGS:--s}" -GO_LDFLAGS="$GO_LDFLAGS -X go.stargrave.org/tofuproxy/warc.UnZSTDPath=$UNZSTD_PATH" -${GO:-go} build -o $3 -ldflags="$GO_LDFLAGS" ./cmd/${1%.cmd} diff --git a/doc/.gitignore b/doc/.gitignore new file mode 100644 index 0000000..2eec3a2 --- /dev/null +++ b/doc/.gitignore @@ -0,0 +1 @@ +/tofuproxy.info diff --git a/doc/download.texi b/doc/download.texi new file mode 100644 index 0000000..854c591 --- /dev/null +++ b/doc/download.texi @@ -0,0 +1,4 @@ +@multitable {XXXXX} {XXXX-XX-XX} {XXXX KiB} {meta4 link sig} {xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx} +@headitem Version @tab Date @tab Size @tab Tarball @tab SHA256 checksum + +@end multitable diff --git a/doc/index.texi b/doc/index.texi index 9c75ae6..2314c9f 100644 --- a/doc/index.texi +++ b/doc/index.texi @@ -103,6 +103,7 @@ Web fonts downloads are forbidden. @insertcopying @include why.texi +@include install.texi @include usage.texi @include spies.texi @include certs.texi diff --git a/doc/install.texi b/doc/install.texi new file mode 100644 index 0000000..4e71352 --- /dev/null +++ b/doc/install.texi @@ -0,0 +1,26 @@ +@node Install +@unnumbered Install + +Download the tarball with the signature from website and build it. +You need more or less modern @url{https://go.dev/, Go} and C-compiler +with @url{https://facebook.github.io/zstd/, libzstd} headers if you need +WARCs support. + +@example +$ [fetch|wget] http://www.tofuproxy.stargrave.org/download/tofuproxy-@value{VERSION}.tar.zst +$ [fetch|wget] http://www.tofuproxy.stargrave.org/download/tofuproxy-@value{VERSION}.tar.zst.sig +$ gpg --verify tofuproxy-@value{VERSION}.tar.zst.sig tofuproxy-@value{VERSION}.tar.zst +$ zstd -d < tofuproxy-@value{VERSION}.tar.zst | tar xf - +$ cd tofuproxy-@value{VERSION} +$ ./build +@end example + +@include download.texi + +You @strong{have to} verify downloaded tarballs integrity and +authenticity to be sure that you retrieved trusted and untampered +software. @url{https://www.gnupg.org/, GNU Privacy Guard} is used +for that purpose. + +Also there is @url{https://yggdrasil-network.github.io/, Yggdrasil} +accessible address: @url{http://y.www.tofuproxy.stargrave.org}. diff --git a/doc/restricted.texi b/doc/restricted.texi index 241b71e..0774315 100644 --- a/doc/restricted.texi +++ b/doc/restricted.texi @@ -7,7 +7,7 @@ CA. For example you want to limit CA with SPKI's SHA256 hash of to domains only in @code{stargrave.org} tree: @example -$ tee fifos/add-restricted < restricted.txt +$ tee fifos/add-restricted < contrib/restricted.txt 9215d9eeddeb403b0ffebb228cfc13104da825117d3640a0dfbfc0c08a012124 stargrave.org [...] @end example diff --git a/doc/spies.texi b/doc/spies.texi index 6760132..1b92372 100644 --- a/doc/spies.texi +++ b/doc/spies.texi @@ -5,7 +5,7 @@ You can reject requests to the whole domains. As a rule some spying ones (for advertisements and better user experience, they say). @example -$ tee fifos/add-spies < spies.txt +$ tee fifos/add-spies < contrib/spies.txt ads.google.com advertising.yandex.ru [...] diff --git a/doc/tofuproxy.info.do b/doc/tofuproxy.info.do new file mode 100644 index 0000000..3df7ba8 --- /dev/null +++ b/doc/tofuproxy.info.do @@ -0,0 +1,7 @@ +redo-ifchange *.texi ../VERSION +${MAKEINFO:=makeinfo} \ + -D "VERSION `cat ../VERSION`" \ + --set-customization-variable SECTION_NAME_IN_TITLE=1 \ + --set-customization-variable TREE_TRANSFORMATIONS=complete_tree_nodes_menus \ + --set-customization-variable ASCII_PUNCTUATION=1 \ + --output $3 index.texi diff --git a/doc/usage.texi b/doc/usage.texi index ba2fdcb..57b0b3d 100644 --- a/doc/usage.texi +++ b/doc/usage.texi @@ -3,40 +3,31 @@ Currently @command{tofuproxy} uses: GnuTLS'es @url{https://www.gnutls.org/manual/html_node/certtool-Invocation.html, certtool}, -@url{http://cr.yp.to/redo.html, redo} build system, @url{https://www.tcl.tk/, Tcl/Tk}'s @command{wish} shell for GUI dialogues, @command{dwebp}, @command{djxl}, @command{avifdec} for images transcoding, @url{https://vanheusden.com/multitail/, multitail} for logs viewing. @itemize -@item Download and build @command{tofuproxy}: - -@example -$ git clone git://git.stargrave.org/tofuproxy.git -$ cd tofuproxy -$ redo all -@end example +@item @ref{Install, Download} and build it. @item -If build fails because of untrusted @code{ca.cypherpunks.ru} certificate, then: +Create separate @file{state/} directory, generate @command{tofuproxy}'s +certificate and create necessary FIFOs by executing @command{state-init} +helper: @example -$ [fetch|wget] http://www.ca.cypherpunks.ru/cert.pem -$ [fetch|wget] http://www.ca.cypherpunks.ru/cert.pem.asc -$ gpg --auto-key-locate dane --locate-keys stargrave at stargrave dot org -$ gpg --auto-key-locate wkd --locate-keys stargrave at gnupg dot net -$ gpg --verify cert.pem.asc -$ SSL_CERT_FILE=`pwd`/cert.pem GIT_SSL_CAINFO=`pwd`/cert.pem redo all +$ mkdir state ; cd state +$ ../state-init @end example @item -Run @command{tofuproxy} itself. By default it will bind to +Run @command{tofuproxy} itself then. By default it will bind to @code{[::1]:8080} and use @code{[::1]:53} DNS server for DANE requests (set to an empty string to disable DANE lookups): @example -$ ./tofuproxy.cmd +$ ../cmd/tofuproxy/tofuproxy [-ai ecdsa] -unzstd ../cmd/zstd/unzstd main.go:70: listening: [::1]:8080 dns: [::1]:53 certs: ./certs ccerts: ./ccerts @end example @@ -51,7 +42,7 @@ main.go:70: listening: [::1]:8080 dns: [::1]:53 certs: ./certs ccerts: ./ccerts @item Watch logs: @example -$ ( cd fifos ; ./multitail.sh ) +$ ( cd fifos ; ../../fifos/multitail ) @end example @image{logs,,,Example logs,.webp} diff --git a/doc/warcs.texi b/doc/warcs.texi index 681354e..7408e91 100644 --- a/doc/warcs.texi +++ b/doc/warcs.texi @@ -78,17 +78,17 @@ save in-memory index to the disk as @file{....idx.gob} files. During the next load, if those files exists, they are used as index immediately, without expensive WARC parsing. -@code{redo warc-extract.cmd} utility uses exactly the same code for -parsing WARCs. It can be used to check if WARCs can be successfully +@code{cmd/warc-extract/warc-extract} utility uses exactly the same code +for parsing WARCs. It can be used to check if WARCs can be successfully loaded, to list all URIs after, to extract some specified URI and to pre-generate @file{.idx.gob} indices. @example -$ warc-extract.cmd -idx \ +$ cmd/warc-extract/warc-extract -idx \ smth.warc-00000.warc.gz \ smth.warc-00001.warc.gz \ smth.warc-00002.warc.gz -$ warc-extract.cmd -uri http://some/uri \ +$ cmd/warc-extract/warc-extract -uri http://some/uri \ smth.warc-00000.warc.gz \ smth.warc-00001.warc.gz \ smth.warc-00002.warc.gz @@ -99,8 +99,7 @@ from any kind of already existing WARCs. It has better compression ratio and much higher decompression speed, than @file{.warc.gz}. @example -$ redo cmd/zstd/enzstd -$ ./warc-extract.cmd -for-enzstd /path/to.warc.gz | +$ cmd/warc-extract/warc-extract -for-enzstd /path/to.warc.gz | cmd/zstd/enzstd > /path/to.warc.zst @end example diff --git a/doc/www.do b/doc/www.do index b9c7ba7..bbf848d 100644 --- a/doc/www.do +++ b/doc/www.do @@ -1,8 +1,9 @@ -redo-ifchange *.texi +redo-ifchange *.texi ../VERSION html=tofuproxy.html rm -f $html/*.html ${MAKEINFO:=makeinfo} --html \ --css-include style.css \ + -D "VERSION `cat ../VERSION`" \ --set-customization-variable SECTION_NAME_IN_TITLE=1 \ --set-customization-variable TREE_TRANSFORMATIONS=complete_tree_nodes_menus \ --set-customization-variable FORMAT_MENU=menu \ diff --git a/fifos/ensure.do b/fifos/ensure old mode 100644 new mode 100755 similarity index 96% rename from fifos/ensure.do rename to fifos/ensure index 03d50d9..cb5709d --- a/fifos/ensure.do +++ b/fifos/ensure @@ -1,3 +1,5 @@ +#!/bin/sh -e + for f in cert dane err http-auth non-ok ok redir req tls tls-auth various warc ; do [ -p log-$f ] || mkfifo log-$f done diff --git a/fifos/multitail.sh b/fifos/multitail similarity index 98% rename from fifos/multitail.sh rename to fifos/multitail index f249f0e..4813786 100755 --- a/fifos/multitail.sh +++ b/fifos/multitail @@ -1,6 +1,6 @@ #!/bin/sh -multitail \ +exec multitail \ -wh 10 \ -t "Trust" -ci magenta -l "while :; do tai64nlocal < log-cert ; done" \ -t "DANE" --label "DANE " -L "while :; do tai64nlocal < log-dane ; done" \ diff --git a/makedist b/makedist new file mode 100755 index 0000000..b44a525 --- /dev/null +++ b/makedist @@ -0,0 +1,82 @@ +#!/bin/sh -ex + +cur=$(pwd) +tmp=$(mktemp -d) +release=$1 +[ -n "$release" ] + +git clone . $tmp/tofuproxy-$release +cd $tmp/tofuproxy-$release +git checkout v$release + +redo-ifchange VERSION + +######################################################################## +cd doc +cat > download.texi < $texi < "$tarball".asc +meta4-create -file "$tarball" -mtime "$tarball" -sig "$tarball".asc \ + http://www.tofuproxy.stargrave.org/download/"$tarball" \ + http://y.www.tofuproxy.stargrave.org/download/"$tarball" > "$tarball".meta4 + +size=$(( $(stat -f %z $tarball) / 1024 )) +hash=$(gpg --print-md SHA256 < $tarball) +release_date=$(date "+%Y-%m-%d") + +release_underscored=`echo $release | tr . _` +cat < cert.pem + umask $prev +} +mkdir -p fifos +cd fifos +"$root"/fifos/ensure +cd .. +for e in "$root"/contrib/*.txt ; do + [ -s "${e##*/}" ] || cp $e . +done diff --git a/version.go b/version.go new file mode 100644 index 0000000..dd009ec --- /dev/null +++ b/version.go @@ -0,0 +1,3 @@ +package tofuproxy + +const Version = "0.1.0" -- 2.44.0