bin/cmd.list | 1 + doc/cmd/index.texi | 2 ++ doc/cmd/nncp-trns.texi | 13 +++++++++++++ doc/download.texi | 4 ++++ doc/multicast.texi | 7 ++----- doc/news.ru.texi | 22 ++++++++++++++++++++++ doc/news.texi | 23 +++++++++++++++++++++++ doc/nncp.html.do | 1 + ports/nncp/Makefile | 2 +- ports/nncp/pkg-plist | 1 + src/check.go | 4 ++-- src/cmd/nncp-hash/main.go | 14 ++++++++++---- src/cmd/nncp-pkt/main.go | 6 ++++-- src/cmd/nncp-trns/main.go | 143 +++++++++++++++++++++++++++++++++++++++++++++++++++++ src/ctx.go | 2 +- src/mth.go | 260 +++++++++++++++++++++++++++++++++++++++++++++--------- src/mth_test.go | 52 ++++++++++++++++++++++++++++++++++++++++++---------- src/nncp.go | 2 +- src/sp.go | 6 +++--- src/toss.go | 54 ++++++++++++++++++++++++++++++++++++++++++++--------- diff --git a/bin/cmd.list b/bin/cmd.list index dd5e9536b997a525d347ed124b0704c0ad10553589eedc5db802db123536264a..714a9f8052125221956bd1f0c3bf514ce4d1284efc4f07ac5517af3bd7508eff 100644 --- a/bin/cmd.list +++ b/bin/cmd.list @@ -17,4 +17,5 @@ nncp-reass nncp-rm nncp-stat nncp-toss +nncp-trns nncp-xfer diff --git a/doc/cmd/index.texi b/doc/cmd/index.texi index cc7f8c639430586589b0c0d8f90f7a13a5b29d8dea88812effaca0180525550f..ccabd646780510828564e67178c164746d3510cb885a443d91b5d8ae9731dc2c 100644 --- a/doc/cmd/index.texi +++ b/doc/cmd/index.texi @@ -54,6 +54,7 @@ * nncp-file:: * nncp-exec:: * nncp-freq:: +* nncp-trns:: Packets sharing commands @@ -88,6 +89,7 @@ @include cmd/nncp-cfgenc.texi @include cmd/nncp-file.texi @include cmd/nncp-exec.texi @include cmd/nncp-freq.texi +@include cmd/nncp-trns.texi @include cmd/nncp-xfer.texi @include cmd/nncp-bundle.texi @include cmd/nncp-toss.texi diff --git a/doc/cmd/nncp-trns.texi b/doc/cmd/nncp-trns.texi new file mode 100644 index 0000000000000000000000000000000000000000..fde75fce4330a7cb3f873e6fe88daea0f01fd4e1b14705d8208562107b73311e --- /dev/null +++ b/doc/cmd/nncp-trns.texi @@ -0,0 +1,13 @@ +@node nncp-trns +@section nncp-trns + +@example +$ nncp-trns [options] -via NODEx[,...] NODE:PKT +$ nncp-trns [options] -via NODEx[,...] /path/to/PKT + +@end example + +Transit specified encrypted packet via another @option{NODEx}es. +Just manual transition packets creator. Normally you should not use that +command at all, preferring automatic wrapping in transitional packets +using the general @option{-via} option and configuration's files one. diff --git a/doc/download.texi b/doc/download.texi index 84a8e3251cb58af1e59b84c251d9d6fe459d70c415632177127feadee214b7b1..3cf15b85219a49547c0192f0e3bc05b140f4fbf298fa7392a7daf0149499dd9c 100644 --- a/doc/download.texi +++ b/doc/download.texi @@ -28,6 +28,10 @@ @multitable {XXXXX} {XXXX-XX-XX} {XXXX KiB} {link sign} {xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx} @headitem Version @tab Date @tab Size @tab Tarball @tab SHA256 checksum +@item @ref{Release 7_1_1, 7.1.1} @tab 2021-07-06 @tab 1132 KiB +@tab @url{download/nncp-7.1.1.tar.xz, link} @url{download/nncp-7.1.1.tar.xz.sig, sign} +@tab @code{B741C9E3 EC3DB342 893FE081 888C40E4 B94E4298 E5C1A8E0 BA4D179C C239CCCA} + @item @ref{Release 7_1_0, 7.1.0} @tab 2021-07-04 @tab 1142 KiB @tab @url{download/nncp-7.1.0.tar.xz, link} @url{download/nncp-7.1.0.tar.xz.sig, sign} @tab @code{D3BC010F 5D86BB59 E07A2A84 2FF9C73B 4C2F780B 807EF25C E4BC477C E40764A6} diff --git a/doc/multicast.texi b/doc/multicast.texi index 51fddd1991e41999e5a1dc352c45bc03b2c38a679a1b2467fba383a3f22c4c2d..8b02ddc66b67e3c6fbf8f8ea30291d651771d654ee1b8994c061f93516b71100 100644 --- a/doc/multicast.texi +++ b/doc/multicast.texi @@ -100,7 +100,7 @@ D @end example @example -$ nncp-file nodelist-20210704.rec area:nodelist-updates: +$ nncp-file nodelist-20210704.rec.zst area:nodelist-updates: $ nncp-toss -node self @end example @@ -126,7 +126,7 @@ not read area's message because it lacks the keys. @item @code{nodeC} does not relay it to anyone. Just stores -@file{nodelist-20210704.rec} in the incoming directory. +@file{nodelist-20210704.rec.zst} in the incoming directory. @item @code{nodeD} receives packets from both @code{nodeA} and @code{nodeB}. @@ -136,9 +136,6 @@ If @code{nodeD} will receive packet from the @code{nodeB} first, it will relay it to the @code{nodeA} also, that will silently remove it when tossing, because it was already seen. - -@strong{TODO}: we must not relay packet to the node also presenting as -the sender of the area's message. Obviously it has seen it. @item When @code{nodeC} sends message to the area, then @code{nodeA} will diff --git a/doc/news.ru.texi b/doc/news.ru.texi index f5d57073699f8ebb49730b4db81f2f502d8a2ad311ddaabfa2bc4f2afc9ba876..c479b62d279bed344f83f484a8e1a97d8df5f522faf048c8aea0a3176d7a253b 100644 --- a/doc/news.ru.texi +++ b/doc/news.ru.texi @@ -1,6 +1,28 @@ @node Новости @section Новости +@node Релиз 7.2.0 +@subsection Релиз 7.2.0 +@itemize + +@item +Появилась @command{nncp-trns} команда для ручного создания транзитных пакетов. + +@item +Если у целевой ноды транзитного пакета задан @option{via} маршрут, то +использовать его, а не игнорировать. + +@item +Не отправлять multicast пакет оригинатору сообщения, очевидно точно +видящего свой собственный пакет. + +@item +Намного меньшее потребление памяти во время MTH хэширования когда +смещение равно нулю: когда пакет не является докачиванием, а например +проверяется @command{nncp-check} командой. + +@end itemize + @node Релиз 7.1.1 @subsection Релиз 7.1.1 @itemize diff --git a/doc/news.texi b/doc/news.texi index 42406a0aaf03e9049d8f336a41083e20f90260ef1192a49dc9031e1b475afd8e..076453f6dba3c3621d98c4013387af9624071cfb83f9f16206599325514668a8 100644 --- a/doc/news.texi +++ b/doc/news.texi @@ -3,12 +3,35 @@ @unnumbered News See also this page @ref{Новости, on russian}. +@node Release 7_2_0 +@section Release 7.2.0 +@itemize + +@item +@command{nncp-trns} command appeared for manual transition packets creation. + +@item +If destination node of transitional packet has non empty @option{via} +route, then do not ignore, but use it. + +@item +Do not relay multicast packet to area message's originator, that +obviously has seen its own packet. + +@item +Much less memory usage during MTH hashing when offset is zero: when +packet is not resumed, but for example checked with @command{nncp-check} +command. + +@end itemize + @node Release 7_1_1 @section Release 7.1.1 @itemize @item Fixed failing directories fsync after @file{.seen} file creation. + @end itemize @node Release 7_1_0 diff --git a/doc/nncp.html.do b/doc/nncp.html.do index 3366cf20ab43f6e7fccffdaf3683ee1cb9b496d0b63ff1f0ab681d286819f07d..a246272b64115dbfbfbaa53270fee68a4923d0180f6be1fe3eb80401823bdcde 100644 --- a/doc/nncp.html.do +++ b/doc/nncp.html.do @@ -1,5 +1,6 @@ rm -fr nncp.html MAKEINFO_OPTS="$MAKEINFO_OPTS --html --css-include style.css" +MAKEINFO_OPTS="$MAKEINFO_OPTS --set-customization-variable FORMAT_MENU=menu" MAKEINFO_OPTS="$MAKEINFO_OPTS --set-customization-variable SHOW_TITLE=0" MAKEINFO_OPTS="$MAKEINFO_OPTS --set-customization-variable DATE_IN_HEADER=1" MAKEINFO_OPTS="$MAKEINFO_OPTS --set-customization-variable TOP_NODE_UP_URL=index.html" diff --git a/ports/nncp/Makefile b/ports/nncp/Makefile index 156d5c03c4b7b1458f8211cdd5cefbc5cad9f1b2ad69b96e662b686a2d20a2d1..3537056669a9f1afe2fbe7a58f99978be3de3a471f85eeed1625d7ca82903eb9 100644 --- a/ports/nncp/Makefile +++ b/ports/nncp/Makefile @@ -1,5 +1,5 @@ PORTNAME= nncp -DISTVERSION= 7.1.0 +DISTVERSION= 7.2.0 CATEGORIES= net MASTER_SITES= http://www.nncpgo.org/download/ diff --git a/ports/nncp/pkg-plist b/ports/nncp/pkg-plist index 1ebe6e12c49dd16377dbdb21b39ceee5195cb8c6c0a9b823a288886059afc452..8774a77ed27fc73309ff83df4a7a5c979eb23b6b6dde2e62fcb6db03ca87454a 100644 --- a/ports/nncp/pkg-plist +++ b/ports/nncp/pkg-plist @@ -17,6 +17,7 @@ bin/nncp-reass bin/nncp-rm bin/nncp-stat bin/nncp-toss +bin/nncp-trns bin/nncp-xfer @dir etc/newsyslog.conf.d @sample etc/nncp.conf.sample etc/newsyslog.conf.d/nncp.conf diff --git a/src/check.go b/src/check.go index cbd2604492b8c5a945e26c7277313fc2c7165526e20785c1ec808fd43bdeef8f..4fca63b5d6953bcf9adff6da73fe784a5f1eb38d048fb0384c55a0ae8c78ad91 100644 --- a/src/check.go +++ b/src/check.go @@ -79,7 +79,7 @@ func (ctx *Ctx) Check(nodeId *NodeId) bool { return !(ctx.checkXxIsBad(nodeId, TRx) || ctx.checkXxIsBad(nodeId, TTx)) } -func (ctx *Ctx) CheckNoCK(nodeId *NodeId, hshValue *[MTHSize]byte, mth *MTH) (int64, error) { +func (ctx *Ctx) CheckNoCK(nodeId *NodeId, hshValue *[MTHSize]byte, mth MTH) (int64, error) { dirToSync := filepath.Join(ctx.Spool, nodeId.String(), string(TRx)) pktName := Base32Codec.EncodeToString(hshValue[:]) pktPath := filepath.Join(dirToSync, pktName) @@ -103,7 +103,7 @@ var gut bool if mth == nil { gut, err = Check(fd, size, hshValue[:], les, ctx.ShowPrgrs) } else { - mth.PktName = pktName + mth.SetPktName(pktName) if _, err = mth.PrependFrom(bufio.NewReaderSize(fd, MTHSize)); err != nil { return 0, err } diff --git a/src/cmd/nncp-hash/main.go b/src/cmd/nncp-hash/main.go index 0b12aedca31d430c899ee90be64c123b3b77024a4715a48db73e6178d262db9b..8b10853e34c5a53f8148735d64fa7251071f1a71766790e4be9a68a2c1aba437 100644 --- a/src/cmd/nncp-hash/main.go +++ b/src/cmd/nncp-hash/main.go @@ -42,6 +42,7 @@ func main() { var ( fn = flag.String("file", "", "Read the file instead of stdin") seek = flag.Uint64("seek", 0, "Seek the file, hash, rewind, hash remaining") + forceFat = flag.Bool("force-fat", false, "Force MTHFat implementation usage") showPrgrs = flag.Bool("progress", false, "Progress showing") debug = flag.Bool("debug", false, "Print MTH steps calculations") version = flag.Bool("version", false, "Print version information") @@ -75,15 +76,20 @@ log.Fatalln(err) } size = fi.Size() } - mth := nncp.MTHNew(size, int64(*seek)) + var mth nncp.MTH + if *forceFat { + mth = nncp.MTHFatNew(size, int64(*seek)) + } else { + mth = nncp.MTHNew(size, int64(*seek)) + } var debugger sync.WaitGroup if *debug { fmt.Println("Leaf BLAKE3 key:", hex.EncodeToString(nncp.MTHLeafKey[:])) fmt.Println("Node BLAKE3 key:", hex.EncodeToString(nncp.MTHNodeKey[:])) - mth.Events = make(chan nncp.MTHEvent) + events := mth.Events() debugger.Add(1) go func() { - for e := range mth.Events { + for e := range events { var t string switch e.Type { case nncp.MTHEventAppend: @@ -121,7 +127,7 @@ if _, err = fd.Seek(0, io.SeekStart); err != nil { log.Fatalln(err) } if *showPrgrs { - mth.PktName = *fn + mth.SetPktName(*fn) } if _, err = mth.PrependFrom(bufio.NewReaderSize(fd, nncp.MTHBlockSize)); err != nil { log.Fatalln(err) diff --git a/src/cmd/nncp-pkt/main.go b/src/cmd/nncp-pkt/main.go index 258543e7d278894a00844234d52642095d8a0f8f02621046fb013bea1b0fd2e0..b4780254db4ccbc56c1f3872e8629a0e525e679aae6769e84029ffcbceaabb2b 100644 --- a/src/cmd/nncp-pkt/main.go +++ b/src/cmd/nncp-pkt/main.go @@ -84,7 +84,7 @@ )) case nncp.PktTypeTrns: path = nncp.Base32Codec.EncodeToString(pkt.Path[:pkt.PathLen]) node, err := ctx.FindNode(path) - if err != nil { + if err == nil { path = fmt.Sprintf("%s (%s)", path, node.Name) } case nncp.PktTypeArea: @@ -119,7 +119,9 @@ var area *nncp.Area recipientNode := ctx.Neigh[*pktEnc.Recipient] if recipientNode == nil { area = ctx.AreaId2Area[nncp.AreaId(*pktEnc.Recipient)] - recipientName = "area " + area.Name + if area != nil { + recipientName = "area " + area.Name + } } else { recipientName = recipientNode.Name } diff --git a/src/cmd/nncp-trns/main.go b/src/cmd/nncp-trns/main.go new file mode 100644 index 0000000000000000000000000000000000000000..a449f0014cef3222674ee45b83ac4a7662590fcb6c9722ed63501c953a9de163 --- /dev/null +++ b/src/cmd/nncp-trns/main.go @@ -0,0 +1,143 @@ +/* +NNCP -- Node to Node copy, utilities for store-and-forward data exchange +Copyright (C) 2016-2021 Sergey Matveev + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, version 3 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// Wrap existing encrypted packet to transition ones. +package main + +import ( + "flag" + "fmt" + "io" + "log" + "os" + "path/filepath" + "strings" + + "go.cypherpunks.ru/nncp/v7" +) + +func usage() { + fmt.Fprintf(os.Stderr, nncp.UsageHeader()) + fmt.Fprintf(os.Stderr, "nncp-trns -- transit existing encrypted packet\n\n") + fmt.Fprintf(os.Stderr, "Usage: %s [options] -via NODEx[,...] NODE:PKT\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " (to transit SPOOL/NODE/tx/PKT)\n") + fmt.Fprintf(os.Stderr, " %s [options] -via NODEx[,...] /path/to/PKT\nOptions:\n", + os.Args[0]) + flag.PrintDefaults() +} + +func main() { + var ( + cfgPath = flag.String("cfg", nncp.DefaultCfgPath, "Path to configuration file") + niceRaw = flag.String("nice", nncp.NicenessFmt(nncp.DefaultNiceFile), "Outbound packet niceness") + viaOverride = flag.String("via", "", "Override Via path to destination node") + spoolPath = flag.String("spool", "", "Override path to spool") + logPath = flag.String("log", "", "Override path to logfile") + quiet = flag.Bool("quiet", false, "Print only errors") + showPrgrs = flag.Bool("progress", false, "Force progress showing") + omitPrgrs = flag.Bool("noprogress", false, "Omit progress showing") + debug = flag.Bool("debug", false, "Print debug messages") + version = flag.Bool("version", false, "Print version information") + warranty = flag.Bool("warranty", false, "Print warranty information") + ) + log.SetFlags(log.Lshortfile) + flag.Usage = usage + flag.Parse() + if *warranty { + fmt.Println(nncp.Warranty) + return + } + if *version { + fmt.Println(nncp.VersionGet()) + return + } + if flag.NArg() != 1 { + usage() + os.Exit(1) + } + nice, err := nncp.NicenessParse(*niceRaw) + if err != nil { + log.Fatalln(err) + } + + ctx, err := nncp.CtxFromCmdline( + *cfgPath, + *spoolPath, + *logPath, + *quiet, + *showPrgrs, + *omitPrgrs, + *debug, + ) + if err != nil { + log.Fatalln("Error during initialization:", err) + } + if ctx.Self == nil { + log.Fatalln("Config lacks private keys") + } + ctx.Umask() + + var pktPath string + var pktName string + if _, err = os.Stat(flag.Arg(0)); err == nil { + pktPath = flag.Arg(0) + pktName = filepath.Base(pktPath) + } else { + splitted := strings.Split(flag.Arg(0), ":") + if len(splitted) != 2 { + log.Fatalln("Invalid NODE:PKT specification") + } + node, err := ctx.FindNode(splitted[0]) + if err != nil { + log.Fatalln("Invalid NODE specified:", err) + } + pktPath = filepath.Join( + ctx.Spool, node.Id.String(), string(nncp.TTx), splitted[1], + ) + pktName = filepath.Base(splitted[1]) + } + + fd, err := os.Open(pktPath) + if err != nil { + log.Fatalln(err) + } + fi, err := fd.Stat() + if err != nil { + log.Fatalln(err) + } + pktEnc, _, err := ctx.HdrRead(fd) + if err != nil { + log.Fatalln(err) + } + if _, err = fd.Seek(0, io.SeekStart); err != nil { + log.Fatalln(err) + } + + node := ctx.Neigh[*pktEnc.Recipient] + nncp.ViaOverride(*viaOverride, ctx, node) + via := node.Via[:len(node.Via)-1] + node = ctx.Neigh[*node.Via[len(node.Via)-1]] + node.Via = via + + pktTrns, err := nncp.NewPkt(nncp.PktTypeTrns, 0, pktEnc.Recipient[:]) + if err != nil { + panic(err) + } + if _, err = ctx.Tx(node, pktTrns, nice, fi.Size(), 0, fd, pktName, nil); err != nil { + log.Fatalln(err) + } +} diff --git a/src/ctx.go b/src/ctx.go index 7500bcd00bd84813b22aed83f7a8f41b95a6af113d2857091dd079cf3484bd6f..84594d56fa651910570032d8344201ee02243c9e49c74b86ee155ec223b0c5eb 100644 --- a/src/ctx.go +++ b/src/ctx.go @@ -138,7 +138,7 @@ func (ctx *Ctx) IsEnoughSpace(want int64) bool { var s unix.Statfs_t if err := unix.Statfs(ctx.Spool, &s); err != nil { - log.Fatalln(err) + log.Fatalln("Can not stat spool:", err) } return int64(s.Bavail)*int64(s.Bsize) > want } diff --git a/src/mth.go b/src/mth.go index be012ec049eed77e82d8e6ced7643e1171737cf1757b728b72cfed9c2d7d2210..8d39d74eb61ad66140dbb11c866ad654ba6a5cf973e2d533238a49c3d0227447 100644 --- a/src/mth.go +++ b/src/mth.go @@ -20,6 +20,7 @@ import ( "bytes" "errors" + "hash" "io" "lukechampine.com/blake3" @@ -50,21 +51,29 @@ Ctr int Hsh []byte } -type MTH struct { +type MTH interface { + hash.Hash + PrependFrom(r io.Reader) (int, error) + SetPktName(n string) + PrependSize() int64 + Events() chan MTHEvent +} + +type MTHFat struct { size int64 - PrependSize int64 + prependSize int64 skip int64 skipped bool hasher *blake3.Hasher hashes [][MTHSize]byte buf *bytes.Buffer finished bool - Events chan MTHEvent - PktName string + events chan MTHEvent + pktName string } -func MTHNew(size, offset int64) *MTH { - mth := MTH{ +func MTHFatNew(size, offset int64) MTH { + mth := MTHFat{ hasher: blake3.New(MTHSize, MTHLeafKey[:]), buf: bytes.NewBuffer(make([]byte, 0, 2*MTHBlockSize)), } @@ -86,19 +95,28 @@ if offset+skip > size { skip = size - offset } mth.size = size - mth.PrependSize = prependSize + mth.prependSize = prependSize mth.skip = skip mth.hashes = make([][MTHSize]byte, prepends, 1+size/MTHBlockSize) return &mth } -func (mth *MTH) Reset() { panic("not implemented") } +func (mth *MTHFat) Events() chan MTHEvent { + mth.events = make(chan MTHEvent) + return mth.events +} + +func (mth *MTHFat) SetPktName(pktName string) { mth.pktName = pktName } + +func (mth *MTHFat) PrependSize() int64 { return mth.prependSize } + +func (mth *MTHFat) Reset() { panic("not implemented") } -func (mth *MTH) Size() int { return MTHSize } +func (mth *MTHFat) Size() int { return MTHSize } -func (mth *MTH) BlockSize() int { return MTHBlockSize } +func (mth *MTHFat) BlockSize() int { return MTHBlockSize } -func (mth *MTH) Write(data []byte) (int, error) { +func (mth *MTHFat) Write(data []byte) (int, error) { if mth.finished { return 0, errors.New("already Sum()ed") } @@ -118,8 +136,8 @@ h := new([MTHSize]byte) mth.hasher.Sum(h[:0]) mth.hasher.Reset() mth.hashes = append(mth.hashes, *h) - if mth.Events != nil { - mth.Events <- MTHEvent{ + if mth.events != nil { + mth.events <- MTHEvent{ MTHEventAppend, 0, len(mth.hashes) - 1, mth.hashes[len(mth.hashes)-1][:], @@ -129,19 +147,19 @@ } return n, err } -func (mth *MTH) PrependFrom(r io.Reader) (int, error) { +func (mth *MTHFat) PrependFrom(r io.Reader) (int, error) { if mth.finished { return 0, errors.New("already Sum()ed") } var err error buf := make([]byte, MTHBlockSize) var i, n, read int - fullsize := mth.PrependSize - les := LEs{{"Pkt", mth.PktName}, {"FullSize", fullsize}, {"Size", 0}} - for mth.PrependSize >= MTHBlockSize { + fullsize := mth.prependSize + les := LEs{{"Pkt", mth.pktName}, {"FullSize", fullsize}, {"Size", 0}} + for mth.prependSize >= MTHBlockSize { n, err = io.ReadFull(r, buf) read += n - mth.PrependSize -= MTHBlockSize + mth.prependSize -= MTHBlockSize if err != nil { return read, err } @@ -150,30 +168,30 @@ panic(err) } mth.hasher.Sum(mth.hashes[i][:0]) mth.hasher.Reset() - if mth.Events != nil { - mth.Events <- MTHEvent{MTHEventPrepend, 0, i, mth.hashes[i][:]} + if mth.events != nil { + mth.events <- MTHEvent{MTHEventPrepend, 0, i, mth.hashes[i][:]} } - if mth.PktName != "" { + if mth.pktName != "" { les[len(les)-1].V = int64(read) Progress("check", les) } i++ } - if mth.PrependSize > 0 { - n, err = io.ReadFull(r, buf[:mth.PrependSize]) + if mth.prependSize > 0 { + n, err = io.ReadFull(r, buf[:mth.prependSize]) read += n if err != nil { return read, err } - if _, err = mth.hasher.Write(buf[:mth.PrependSize]); err != nil { + if _, err = mth.hasher.Write(buf[:mth.prependSize]); err != nil { panic(err) } mth.hasher.Sum(mth.hashes[i][:0]) mth.hasher.Reset() - if mth.Events != nil { - mth.Events <- MTHEvent{MTHEventPrepend, 0, i, mth.hashes[i][:]} + if mth.events != nil { + mth.events <- MTHEvent{MTHEventPrepend, 0, i, mth.hashes[i][:]} } - if mth.PktName != "" { + if mth.pktName != "" { les[len(les)-1].V = fullsize Progress("check", les) } @@ -181,7 +199,7 @@ } return read, nil } -func (mth *MTH) Sum(b []byte) []byte { +func (mth *MTHFat) Sum(b []byte) []byte { if mth.finished { return append(b, mth.hashes[0][:]...) } @@ -194,8 +212,8 @@ h := new([MTHSize]byte) mth.hasher.Sum(h[:0]) mth.hasher.Reset() mth.hashes = append(mth.hashes, *h) - if mth.Events != nil { - mth.Events <- MTHEvent{ + if mth.events != nil { + mth.events <- MTHEvent{ MTHEventAppend, 0, len(mth.hashes) - 1, mth.hashes[len(mth.hashes)-1][:], @@ -211,14 +229,14 @@ } mth.hasher.Sum(h[:0]) mth.hasher.Reset() mth.hashes = append(mth.hashes, *h) - if mth.Events != nil { - mth.Events <- MTHEvent{MTHEventAppend, 0, 0, mth.hashes[0][:]} + if mth.events != nil { + mth.events <- MTHEvent{MTHEventAppend, 0, 0, mth.hashes[0][:]} } fallthrough case 1: mth.hashes = append(mth.hashes, mth.hashes[0]) - if mth.Events != nil { - mth.Events <- MTHEvent{MTHEventAppend, 0, 1, mth.hashes[1][:]} + if mth.events != nil { + mth.events <- MTHEvent{MTHEventAppend, 0, 1, mth.hashes[1][:]} } } mth.hasher = blake3.New(MTHSize, MTHNodeKey[:]) @@ -237,8 +255,8 @@ h := new([MTHSize]byte) mth.hasher.Sum(h[:0]) mth.hasher.Reset() hashesUp = append(hashesUp, *h) - if mth.Events != nil { - mth.Events <- MTHEvent{ + if mth.events != nil { + mth.events <- MTHEvent{ MTHEventFold, level, len(hashesUp) - 1, hashesUp[len(hashesUp)-1][:], @@ -247,8 +265,8 @@ } } if len(mth.hashes)%2 == 1 { hashesUp = append(hashesUp, mth.hashes[len(mth.hashes)-1]) - if mth.Events != nil { - mth.Events <- MTHEvent{ + if mth.events != nil { + mth.events <- MTHEvent{ MTHEventAppend, level, len(hashesUp) - 1, hashesUp[len(hashesUp)-1][:], @@ -259,8 +277,170 @@ mth.hashes = hashesUp level++ } mth.finished = true - if mth.Events != nil { - close(mth.Events) + if mth.events != nil { + close(mth.events) } return append(b, mth.hashes[0][:]...) } + +type MTHSeqEnt struct { + l int + h [MTHSize]byte +} + +type MTHSeq struct { + hasherLeaf *blake3.Hasher + hasherNode *blake3.Hasher + hashes []MTHSeqEnt + buf *bytes.Buffer + events chan MTHEvent + ctrs []int + finished bool +} + +func MTHSeqNew() *MTHSeq { + mth := MTHSeq{ + hasherLeaf: blake3.New(MTHSize, MTHLeafKey[:]), + hasherNode: blake3.New(MTHSize, MTHNodeKey[:]), + buf: bytes.NewBuffer(make([]byte, 0, 2*MTHBlockSize)), + ctrs: make([]int, 1, 2), + } + return &mth +} + +func (mth *MTHSeq) Reset() { panic("not implemented") } + +func (mth *MTHSeq) Size() int { return MTHSize } + +func (mth *MTHSeq) BlockSize() int { return MTHBlockSize } + +func (mth *MTHSeq) PrependFrom(r io.Reader) (int, error) { + panic("must not reach that code") +} + +func (mth *MTHSeq) Events() chan MTHEvent { + mth.events = make(chan MTHEvent) + return mth.events +} + +func (mth *MTHSeq) SetPktName(pktName string) {} + +func (mth *MTHSeq) PrependSize() int64 { return 0 } + +func (mth *MTHSeq) leafAdd() { + ent := MTHSeqEnt{l: 0} + mth.hasherLeaf.Sum(ent.h[:0]) + mth.hasherLeaf.Reset() + mth.hashes = append(mth.hashes, ent) + if mth.events != nil { + mth.events <- MTHEvent{ + MTHEventAppend, 0, mth.ctrs[0], + mth.hashes[len(mth.hashes)-1].h[:], + } + } + mth.ctrs[0]++ +} + +func (mth *MTHSeq) incr(l int) { + if len(mth.ctrs) <= l { + mth.ctrs = append(mth.ctrs, 0) + } else { + mth.ctrs[l]++ + } +} + +func (mth *MTHSeq) fold() { + for len(mth.hashes) >= 2 { + if mth.hashes[len(mth.hashes)-2].l != mth.hashes[len(mth.hashes)-1].l { + break + } + if _, err := mth.hasherNode.Write(mth.hashes[len(mth.hashes)-2].h[:]); err != nil { + panic(err) + } + if _, err := mth.hasherNode.Write(mth.hashes[len(mth.hashes)-1].h[:]); err != nil { + panic(err) + } + mth.hashes = mth.hashes[:len(mth.hashes)-1] + end := &mth.hashes[len(mth.hashes)-1] + end.l++ + mth.incr(end.l) + mth.hasherNode.Sum(end.h[:0]) + mth.hasherNode.Reset() + if mth.events != nil { + mth.events <- MTHEvent{MTHEventFold, end.l, mth.ctrs[end.l], end.h[:]} + } + } +} + +func (mth *MTHSeq) Write(data []byte) (int, error) { + if mth.finished { + return 0, errors.New("already Sum()ed") + } + n, err := mth.buf.Write(data) + if err != nil { + return n, err + } + for mth.buf.Len() >= MTHBlockSize { + if _, err = mth.hasherLeaf.Write(mth.buf.Next(MTHBlockSize)); err != nil { + return n, err + } + mth.leafAdd() + mth.fold() + } + return n, err +} + +func (mth *MTHSeq) Sum(b []byte) []byte { + if mth.finished { + return append(b, mth.hashes[0].h[:]...) + } + if mth.buf.Len() > 0 { + if _, err := mth.hasherLeaf.Write(mth.buf.Next(MTHBlockSize)); err != nil { + panic(err) + } + mth.leafAdd() + mth.fold() + } + switch mth.ctrs[0] { + case 0: + if _, err := mth.hasherLeaf.Write(nil); err != nil { + panic(err) + } + mth.leafAdd() + fallthrough + case 1: + mth.hashes = append(mth.hashes, mth.hashes[0]) + mth.ctrs[0]++ + if mth.events != nil { + mth.events <- MTHEvent{ + MTHEventAppend, 0, mth.ctrs[0], + mth.hashes[len(mth.hashes)-1].h[:], + } + } + mth.fold() + } + for len(mth.hashes) >= 2 { + l := mth.hashes[len(mth.hashes)-2].l + mth.incr(l) + mth.hashes[len(mth.hashes)-1].l = l + if mth.events != nil { + mth.events <- MTHEvent{ + MTHEventAppend, l, mth.ctrs[l], + mth.hashes[len(mth.hashes)-1].h[:], + } + } + mth.fold() + } + mth.finished = true + if mth.events != nil { + close(mth.events) + } + return append(b, mth.hashes[0].h[:]...) +} + +func MTHNew(size, offset int64) MTH { + if offset == 0 { + return MTHSeqNew() + } + return MTHFatNew(size, offset) +} diff --git a/src/mth_test.go b/src/mth_test.go index 0d48d5421f0d90eedbf68cf80e75b2021041b1334c110f4c8549584cb0c3b6a1..ee9b58ada83ccb240e68c34435ec2ae53dc85b50fc4d8f11b472e64d7c02431d 100644 --- a/src/mth_test.go +++ b/src/mth_test.go @@ -26,7 +26,7 @@ "lukechampine.com/blake3" ) -func TestMTHSymmetric(t *testing.T) { +func TestMTHFatSymmetric(t *testing.T) { xof := blake3.New(32, nil).XOF() f := func(size uint32, offset uint32) bool { size %= 2 * 1024 * 1024 @@ -36,13 +36,13 @@ panic(err) } offset = offset % size - mth := MTHNew(int64(size), 0) + mth := MTHFatNew(int64(size), 0) if _, err := io.Copy(mth, bytes.NewReader(data)); err != nil { panic(err) } hsh0 := mth.Sum(nil) - mth = MTHNew(int64(size), int64(offset)) + mth = MTHFatNew(int64(size), int64(offset)) if _, err := io.Copy(mth, bytes.NewReader(data[int(offset):])); err != nil { panic(err) } @@ -53,14 +53,14 @@ if bytes.Compare(hsh0, mth.Sum(nil)) != 0 { return false } - mth = MTHNew(0, 0) + mth = MTHFatNew(0, 0) mth.Write(data) if bytes.Compare(hsh0, mth.Sum(nil)) != 0 { return false } data = append(data, 0) - mth = MTHNew(int64(size)+1, 0) + mth = MTHFatNew(int64(size)+1, 0) if _, err := io.Copy(mth, bytes.NewReader(data)); err != nil { panic(err) } @@ -69,7 +69,7 @@ if bytes.Compare(hsh0, hsh00) == 0 { return false } - mth = MTHNew(int64(size)+1, int64(offset)) + mth = MTHFatNew(int64(size)+1, int64(offset)) if _, err := io.Copy(mth, bytes.NewReader(data[int(offset):])); err != nil { panic(err) } @@ -80,7 +80,7 @@ if bytes.Compare(hsh00, mth.Sum(nil)) != 0 { return false } - mth = MTHNew(0, 0) + mth = MTHFatNew(0, 0) mth.Write(data) if bytes.Compare(hsh00, mth.Sum(nil)) != 0 { return false @@ -93,10 +93,42 @@ t.Error(err) } } +func TestMTHSeqAndFatEqual(t *testing.T) { + xof := blake3.New(32, nil).XOF() + f := func(size uint32, offset uint32) bool { + size %= 10 * 1024 * 1024 + data := make([]byte, int(size), int(size)+1) + if _, err := io.ReadFull(xof, data); err != nil { + panic(err) + } + fat := MTHFatNew(int64(size), 0) + if _, err := io.Copy(fat, bytes.NewReader(data)); err != nil { + panic(err) + } + hshFat := fat.Sum(nil) + seq := MTHSeqNew() + if _, err := io.Copy(seq, bytes.NewReader(data)); err != nil { + panic(err) + } + return bytes.Compare(hshFat, seq.Sum(nil)) == 0 + } + if err := quick.Check(f, nil); err != nil { + t.Error(err) + } +} + func TestMTHNull(t *testing.T) { - mth := MTHNew(0, 0) - if _, err := mth.Write(nil); err != nil { + fat := MTHFatNew(0, 0) + if _, err := fat.Write(nil); err != nil { + t.Error(err) + } + hshFat := fat.Sum(nil) + + seq := MTHSeqNew() + if _, err := seq.Write(nil); err != nil { t.Error(err) } - mth.Sum(nil) + if bytes.Compare(hshFat, seq.Sum(nil)) != 0 { + t.FailNow() + } } diff --git a/src/nncp.go b/src/nncp.go index 53a83deea72819244c6d205fee4e7d2a4c770a26c274ddb2358b4f121b958c0b..50368596ec27dd60c2d9001673487c73bd25dd03194b617820f1e9a667b11fd1 100644 --- a/src/nncp.go +++ b/src/nncp.go @@ -40,7 +40,7 @@ const Base32Encoded32Len = 52 var ( - Version string = "7.1.1" + Version string = "7.2.0" Base32Codec *base32.Encoding = base32.StdEncoding.WithPadding(base32.NoPadding) ) diff --git a/src/sp.go b/src/sp.go index fa524304feb105b8b1cc0519c60a108700877372acda1c65e08dd9b110b55648..eac64eaa7319eb911f0a243b130fc92aae8c12f4cf00e0077dc0bdc9478d67e9 100644 --- a/src/sp.go +++ b/src/sp.go @@ -41,14 +41,14 @@ SPHeadOverhead = 4 ) type MTHAndOffset struct { - mth *MTH + mth MTH offset uint64 } type SPCheckerTask struct { nodeId *NodeId hsh *[MTHSize]byte - mth *MTH + mth MTH done chan []byte } @@ -1439,7 +1439,7 @@ continue } if hasherAndOffset != nil { delete(state.fileHashers, filePath) - if hasherAndOffset.mth.PrependSize == 0 { + if hasherAndOffset.mth.PrependSize() == 0 { if bytes.Compare(hasherAndOffset.mth.Sum(nil), file.Hash[:]) != 0 { state.Ctx.LogE( "sp-file-bad-checksum", lesp, diff --git a/src/toss.go b/src/toss.go index a95385af8b763b01d2312b666e3af0f0d410813a93e94442196dea67b19f3bab..74f9d98a43f94d9bd086cf3579de49ea4f4dba696db9da87039b19900fa7597d 100644 --- a/src/toss.go +++ b/src/toss.go @@ -588,11 +588,35 @@ return err } ctx.LogD("rx-tx", les, logMsg) if !dryRun { - if err = ctx.TxTrns(node, nice, int64(pktSize), pipeR); err != nil { - ctx.LogE("rx", les, err, func(les LEs) string { - return logMsg(les) + ": txing" - }) - return err + if len(node.Via) == 0 { + if err = ctx.TxTrns(node, nice, int64(pktSize), pipeR); err != nil { + ctx.LogE("rx", les, err, func(les LEs) string { + return logMsg(les) + ": txing" + }) + return err + } + } else { + via := node.Via[:len(node.Via)-1] + node = ctx.Neigh[*node.Via[len(node.Via)-1]] + node = &Node{Id: node.Id, Via: via, ExchPub: node.ExchPub} + pktTrns, err := NewPkt(PktTypeTrns, 0, nodeId[:]) + if err != nil { + panic(err) + } + if _, err = ctx.Tx( + node, + pktTrns, + nice, + int64(pktSize), 0, + pipeR, + pktName, + nil, + ); err != nil { + ctx.LogE("rx", les, err, func(les LEs) string { + return logMsg(les) + ": txing" + }) + return err + } } } ctx.LogI("rx", les, func(les LEs) string { @@ -705,7 +729,7 @@ return logMsgNode(les) + ": already sent" }) continue } - if nodeId != sender.Id { + if nodeId != sender.Id && nodeId != pktEnc.Sender { ctx.LogI("rx-area-echo", lesEcho, logMsgNode) if _, err = ctx.Tx( node, &pkt, nice, int64(pktSize), 0, fullPipeR, pktName, nil, @@ -820,7 +844,7 @@ nil, ) if err != nil { pipeW.CloseWithError(err) - go func() { <-errs }() + <-errs return err } pipeW.Close() @@ -917,6 +941,18 @@ }) isBad = true continue } + sender := ctx.Neigh[*job.PktEnc.Sender] + if sender == nil { + err := errors.New("unknown node") + ctx.LogE("rx-open", les, err, func(les LEs) string { + return fmt.Sprintf( + "Tossing %s/%s", + ctx.NodeName(job.PktEnc.Sender), pktName, + ) + }) + isBad = true + continue + } errs := make(chan error, 1) var sharedKey []byte Retry: @@ -927,7 +963,7 @@ ctx, pipeR, pktName, les, - ctx.Neigh[*job.PktEnc.Sender], + sender, job.PktEnc.Nice, uint64(pktSizeWithoutEnc(job.Size)), job.Path, @@ -955,7 +991,7 @@ if err != nil { isBad = true fd.Close() - go func() { <-errs }() + <-errs continue } if err = <-errs; err == JobRepeatProcess {