bin/.gitignore | 1 + bin/all.do | 2 +- bin/clean.do | 2 +- bin/hjson-cli.do | 3 +++ doc/building.texi | 3 ++- doc/cfg/index.texi | 8 ++++++++ doc/download.texi | 4 ++++ doc/news.ru.texi | 14 ++++++++++++++ doc/news.texi | 13 +++++++++++++ src/check.go | 6 ++++-- src/cmd/nncp-hash/main.go | 87 +++++++++++++++++++++++++++++------------------------ src/mth.go | 495 ++++++++++++++++++++++++----------------------------- src/mth_test.go | 26 +++++++++++++------------- src/nncp.go | 2 +- src/sp.go | 17 ++++++++++------- diff --git a/bin/.gitignore b/bin/.gitignore index 0ff98e802fedc6095a3d05a21f0a09e6e87cb853f455ab4bd7821f9162e872f1..3326391b8d7d263ffc93d432cec70bf6e34ba57c4a631daac83286f4059e3c31 100644 --- a/bin/.gitignore +++ b/bin/.gitignore @@ -1 +1,2 @@ +hjson-cli nncp-* diff --git a/bin/all.do b/bin/all.do index f0c6d1a83e97e73f608e140d14d2668ad047b8d5d0b92148f87a6c3ccbfb4c57..36ac5d4c530a7c63bcd0006f1815aa20427f516b5200ceffbf181d4f7518c4c2 100644 --- a/bin/all.do +++ b/bin/all.do @@ -1,2 +1,2 @@ -redo-ifchange cmd.list +redo-ifchange cmd.list hjson-cli redo-ifchange `cat cmd.list` diff --git a/bin/clean.do b/bin/clean.do index f128b8679a5e7c7a7dce4624f66f195e1d97113f5fbc23ad0bffc2643609b7e2..3dfcfa366b6ca6e89ad5071b2ae059a64a372d226c1a49085f69e3366c18f106 100644 --- a/bin/clean.do +++ b/bin/clean.do @@ -1 +1 @@ -rm -f `cat cmd.list` +rm -f `cat cmd.list` hjson-cli diff --git a/bin/hjson-cli.do b/bin/hjson-cli.do new file mode 100644 index 0000000000000000000000000000000000000000..6273d4e83b90a2c4da9fa4762c09f7eb9bf974dc0a89b7aca7871383d9d95d0a --- /dev/null +++ b/bin/hjson-cli.do @@ -0,0 +1,3 @@ +cd ../src +GO=${GO:-go} +$GO build -o ../bin/$3 github.com/hjson/hjson-go/hjson-cli diff --git a/doc/building.texi b/doc/building.texi index b51a047c4f19d9569c68261f552d55747c081b331739b66ecacf35c2b314e9fc..83f0a52f97c27dea31f938f908e6476cef7524d4cf4ebffdb9fddee37ca182f3 100644 --- a/doc/building.texi +++ b/doc/building.texi @@ -17,6 +17,7 @@ $ gpg --verify nncp-@value{VERSION}.tar.xz.sig nncp-@value{VERSION}.tar.xz $ xz --decompress --stdout nncp-@value{VERSION}.tar.xz | tar xf - $ cd nncp-@value{VERSION} $ redo all +# look for bin/nncp-* binaries (and possibly hjson-cli one) @end example It uses @url{http://cr.yp.to/redo.html, redo} build system for that @@ -28,7 +29,7 @@ @url{http://www.goredo.cypherpunks.ru/, goredo} (NNCP's author creation), @url{https://redo.readthedocs.io/, apenwarr/redo} (@code{contrib/do} is from that project), @url{https://github.com/leahneukirchen/redo-c, redo-c}. -There is @command{install} make-target respecting @env{DESTDIR}. It will +There is @command{install} target respecting @env{DESTDIR}. It will install binaries and info-documentation: @example diff --git a/doc/cfg/index.texi b/doc/cfg/index.texi index 5c6cc712c1bc92b520d01e1b9a7d413bfc5bd3947c140228e193acb33577bb26..2eb251879b2dab81ca7c7ea3cf300e1d0c3f535ee6d5e7d914287cbf694c1b38 100644 --- a/doc/cfg/index.texi +++ b/doc/cfg/index.texi @@ -30,6 +30,14 @@ } } @end verbatim +Do not forget that Hjson can be safely converted to JSON and vice versa +(loosing formatting and comments of course). By default +@command{hjson-cli} utility from @code{github.com/hjson/hjson-go} is +built together with @command{nncp-*} commands too. For querying +information from the JSON from the command line you can use +@code{github.com/itchyny/gojq} and @code{github.com/skanehira/gjo} for +building it up. + And for being able to communicate with at least one other node, you just need to add single key to the @code{neigh} section similar to the "self". diff --git a/doc/download.texi b/doc/download.texi index a247b3d8ff1791d7ae4ddf409edcada53967cb583d2ff1ba3e35c501747ce61c..5658ac0441555905902d830c064d17b76e9723a95d6a8ba86b6b952f53efd933 100644 --- a/doc/download.texi +++ b/doc/download.texi @@ -28,6 +28,10 @@ @multitable {XXXXX} {XXXX-XX-XX} {XXXX KiB} {link sign} {xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx} @headitem Version @tab Date @tab Size @tab Tarball @tab SHA256 checksum +@item @ref{Release 7_3_0, 7.3.0} @tab 2021-07-10 @tab 1141 KiB +@tab @url{download/nncp-7.3.0.tar.xz, link} @url{download/nncp-7.3.0.tar.xz.sig, sign} +@tab @code{CB34487A 6D7EF507 04D4B8F9 5A16EF16 CC841D3D 7F5423B1 EBB7979D 1062EB4E} + @item @ref{Release 7_2_1, 7.2.1} @tab 2021-07-09 @tab 1139 KiB @tab @url{download/nncp-7.2.1.tar.xz, link} @url{download/nncp-7.2.1.tar.xz.sig, sign} @tab @code{6462BA44 7DB30234 DA6DFB4B B5BF890F 6CA2CC36 697B3AE7 E6F86B86 94AC97D6} diff --git a/doc/news.ru.texi b/doc/news.ru.texi index 7a78ae8f0ff9b119e77bdd0716f5567f07e025379a077f24d4c830c40504ac6f..98bbba5f4f6419815bb3956ee745e1ded687e54cf677be931a487fc299eefa44 100644 --- a/doc/news.ru.texi +++ b/doc/news.ru.texi @@ -2,6 +2,7 @@ @node Новости @section Новости @menu +* Релиз 7.3.1:: * Релиз 7.3.0:: * Релиз 7.2.1:: * Релиз 7.2.0:: @@ -52,6 +53,19 @@ * Релиз 0.4:: * Релиз 0.3:: * Релиз 0.2:: @end menu + +@node Релиз 7.3.1 +@subsection Релиз 7.3.1 +@itemize + +@item +Исправлена проблема с возможно остающимся открытым файловым +дескриптором в online командах. + +@item +Существенно снижено потребление памяти MTH хэширования. + +@end itemize @node Релиз 7.3.0 @subsection Релиз 7.3.0 diff --git a/doc/news.texi b/doc/news.texi index 362981100ffc17bde85a54dcc2f53a7529c91631b532b5007f79a3bb6d279b1e..9fe9367a5d433cb128e99157f631fd668143d201135fe077e53b86260a351523 100644 --- a/doc/news.texi +++ b/doc/news.texi @@ -4,6 +4,7 @@ See also this page @ref{Новости, on russian}. @menu +* Release 7.3.1: Release 7_3_1. * Release 7.3.0: Release 7_3_0. * Release 7.2.1: Release 7_2_1. * Release 7.2.0: Release 7_2_0. @@ -54,6 +55,18 @@ * Release 0.4: Release 0_4. * Release 0.3: Release 0_3. * Release 0.2: Release 0_2. @end menu + +@node Release 7_3_1 +@section Release 7.3.1 +@itemize + +@item +Fixed possibly left opened file descriptor in online commands. + +@item +Severely decreased memory usage of MTH hashing. + +@end itemize @node Release 7_3_0 @section Release 7.3.0 diff --git a/src/check.go b/src/check.go index 4fca63b5d6953bcf9adff6da73fe784a5f1eb38d048fb0384c55a0ae8c78ad91..3ef3ef3bfdb393118ab3d4c1e98d5bba2d317bb94f6ada29a1c67dc1125384b2 100644 --- a/src/check.go +++ b/src/check.go @@ -103,8 +103,10 @@ var gut bool if mth == nil { gut, err = Check(fd, size, hshValue[:], les, ctx.ShowPrgrs) } else { - mth.SetPktName(pktName) - if _, err = mth.PrependFrom(bufio.NewReaderSize(fd, MTHSize)); err != nil { + if _, err = mth.PreaddFrom( + bufio.NewReaderSize(fd, MTHSize), + pktName, ctx.ShowPrgrs, + ); err != nil { return 0, err } if bytes.Compare(mth.Sum(nil), hshValue[:]) == 0 { diff --git a/src/cmd/nncp-hash/main.go b/src/cmd/nncp-hash/main.go index 8b10853e34c5a53f8148735d64fa7251071f1a71766790e4be9a68a2c1aba437..33a7fc6179e428076a2a16f135b4a25d41a54e7b4bca2e8223d29c395fa4057b 100644 --- a/src/cmd/nncp-hash/main.go +++ b/src/cmd/nncp-hash/main.go @@ -76,64 +76,71 @@ log.Fatalln(err) } size = fi.Size() } - var mth nncp.MTH - if *forceFat { - mth = nncp.MTHFatNew(size, int64(*seek)) - } else { - mth = nncp.MTHNew(size, int64(*seek)) - } - var debugger sync.WaitGroup + if *debug { fmt.Println("Leaf BLAKE3 key:", hex.EncodeToString(nncp.MTHLeafKey[:])) fmt.Println("Node BLAKE3 key:", hex.EncodeToString(nncp.MTHNodeKey[:])) - events := mth.Events() + } + + var debugger sync.WaitGroup + startDebug := func(events chan nncp.MTHEvent) { debugger.Add(1) go func() { for e := range events { - var t string - switch e.Type { - case nncp.MTHEventAppend: - t = "Add" - case nncp.MTHEventPrepend: - t = "Pre" - case nncp.MTHEventFold: - t = "Fold" - } - fmt.Printf( - "%s\t%03d\t%06d\t%s\n", - t, e.Level, e.Ctr, hex.EncodeToString(e.Hsh), - ) + fmt.Println(e.String()) } debugger.Done() }() } - if *seek != 0 { - if *fn == "" { - log.Fatalln("-file is required with -seek") + copier := func(w io.Writer) error { + _, err := nncp.CopyProgressed( + w, bufio.NewReaderSize(fd, nncp.MTHBlockSize), "hash", + nncp.LEs{{K: "Pkt", V: *fn}, {K: "FullSize", V: size - int64(*seek)}}, + *showPrgrs, + ) + return err + } + + var sum []byte + if *forceFat { + mth := nncp.MTHFatNew() + if *debug { + startDebug(mth.Events()) + } - if _, err = fd.Seek(int64(*seek), io.SeekStart); err != nil { + if err = copier(mth); err != nil { log.Fatalln(err) } - } - if _, err = nncp.CopyProgressed( - mth, bufio.NewReaderSize(fd, nncp.MTHBlockSize), - "hash", nncp.LEs{{K: "Pkt", V: *fn}, {K: "FullSize", V: size - int64(*seek)}}, - *showPrgrs, - ); err != nil { - log.Fatalln(err) - } - if *seek != 0 { - if _, err = fd.Seek(0, io.SeekStart); err != nil { - log.Fatalln(err) + sum = mth.Sum(nil) + } else { + mth := nncp.MTHSeqNew(size, int64(*seek)) + if *debug { + startDebug(mth.Events()) } - if *showPrgrs { - mth.SetPktName(*fn) + if *seek != 0 { + if *fn == "" { + log.Fatalln("-file is required with -seek") + } + if _, err = fd.Seek(int64(*seek), io.SeekStart); err != nil { + log.Fatalln(err) + } } - if _, err = mth.PrependFrom(bufio.NewReaderSize(fd, nncp.MTHBlockSize)); err != nil { + if err = copier(mth); err != nil { log.Fatalln(err) } + if *seek != 0 { + if _, err = fd.Seek(0, io.SeekStart); err != nil { + log.Fatalln(err) + } + if _, err = mth.PreaddFrom( + bufio.NewReaderSize(fd, nncp.MTHBlockSize), + *fn, *showPrgrs, + ); err != nil { + log.Fatalln(err) + } + } + sum = mth.Sum(nil) } - sum := mth.Sum(nil) debugger.Wait() fmt.Println(hex.EncodeToString(sum)) } diff --git a/src/mth.go b/src/mth.go index 82e70ec47e0174759cac7524110d8f4653c192bcf88fa817edaa8495096a360c..4572335d62ddb9d1b9458a4c08a6128ce6b1508182191107957ca63caeb59fef 100644 --- a/src/mth.go +++ b/src/mth.go @@ -19,7 +19,9 @@ package nncp import ( "bytes" + "encoding/hex" "errors" + "fmt" "hash" "io" @@ -36,87 +38,168 @@ MTHLeafKey = blake3.Sum256([]byte("NNCP MTH LEAF")) MTHNodeKey = blake3.Sum256([]byte("NNCP MTH NODE")) ) -type MTHEventType uint8 +type MTHSeqEnt struct { + l int + c int64 + h [MTHSize]byte +} + +func (ent *MTHSeqEnt) String() string { + return fmt.Sprintf("%03d\t%06d\t%s", ent.l, ent.c, hex.EncodeToString(ent.h[:])) +} + +type MTHEventType string const ( - MTHEventAppend MTHEventType = iota - MTHEventPrepend MTHEventType = iota - MTHEventFold MTHEventType = iota + MTHEventAdd MTHEventType = "Add" + MTHEventPreadd MTHEventType = "Pre" + MTHEventFold MTHEventType = "Fold" ) type MTHEvent struct { - Type MTHEventType - Level int64 - Ctr int64 - Hsh []byte + Type MTHEventType + Ent *MTHSeqEnt +} + +func (e MTHEvent) String() string { + return fmt.Sprintf("%s\t%s", e.Type, e.Ent.String()) } type MTH interface { hash.Hash - PrependFrom(r io.Reader) (int64, error) - SetPktName(n string) - PrependSize() int64 + PreaddFrom(r io.Reader, pktName string, showPrgrs bool) (int64, error) + PreaddSize() int64 Events() chan MTHEvent } -type MTHFat struct { +type MTHSeq struct { + hasherLeaf *blake3.Hasher + hasherNode *blake3.Hasher + hashes []MTHSeqEnt + buf *bytes.Buffer + events chan MTHEvent + ctr int64 size int64 prependSize int64 - skip int64 + toSkip int64 skipped bool - hasher *blake3.Hasher - hashes [][MTHSize]byte - buf *bytes.Buffer finished bool - events chan MTHEvent pktName string } -func MTHFatNew(size, offset int64) MTH { - mth := MTHFat{ - hasher: blake3.New(MTHSize, MTHLeafKey[:]), - buf: bytes.NewBuffer(make([]byte, 0, 2*MTHBlockSize)), +func MTHSeqNew(size, offset int64) *MTHSeq { + mth := MTHSeq{ + hasherLeaf: blake3.New(MTHSize, MTHLeafKey[:]), + hasherNode: blake3.New(MTHSize, MTHNodeKey[:]), + buf: bytes.NewBuffer(make([]byte, 0, 2*MTHBlockSize)), } if size == 0 { return &mth } prepends := offset / MTHBlockSize - skip := MTHBlockSize - (offset - prepends*MTHBlockSize) - if skip == MTHBlockSize { - skip = 0 - } else if skip > 0 { + toSkip := MTHBlockSize - (offset - prepends*MTHBlockSize) + if toSkip == MTHBlockSize { + toSkip = 0 + } else if toSkip > 0 { prepends++ } prependSize := prepends * MTHBlockSize + mth.ctr = prepends if prependSize > size { prependSize = size } - if offset+skip > size { - skip = size - offset + if offset+toSkip > size { + toSkip = size - offset } mth.size = size mth.prependSize = prependSize - mth.skip = skip - mth.hashes = make([][MTHSize]byte, prepends, 1+size/MTHBlockSize) + mth.toSkip = toSkip return &mth } -func (mth *MTHFat) Events() chan MTHEvent { +func (mth *MTHSeq) Reset() { panic("not implemented") } + +func (mth *MTHSeq) Size() int { return MTHSize } + +func (mth *MTHSeq) BlockSize() int { return MTHBlockSize } + +func (mth *MTHSeq) PreaddFrom(r io.Reader, pktName string, showPrgrs bool) (int64, error) { + if mth.finished { + return 0, errors.New("already Sum()ed") + } + if mth.buf.Len() > 0 { + if _, err := mth.hasherLeaf.Write(mth.buf.Next(MTHBlockSize)); err != nil { + panic(err) + } + mth.leafAdd() + mth.fold() + } + prevHashes := mth.hashes + mth.hashes = nil + prevCtr := mth.ctr + mth.ctr = 0 + lr := io.LimitedReader{R: r, N: mth.prependSize} + les := LEs{{"Pkt", pktName}, {"FullSize", mth.prependSize}} + n, err := CopyProgressed(mth, &lr, "prehash", les, showPrgrs) + for _, ent := range prevHashes { + mth.hashes = append(mth.hashes, ent) + mth.fold() + } + if mth.buf.Len() > 0 { + mth.ctr = prevCtr - 1 + } else { + mth.ctr = prevCtr + } + return n, err +} + +func (mth *MTHSeq) Events() chan MTHEvent { mth.events = make(chan MTHEvent) return mth.events } -func (mth *MTHFat) SetPktName(pktName string) { mth.pktName = pktName } +func (mth *MTHSeq) PreaddSize() int64 { return mth.prependSize } -func (mth *MTHFat) PrependSize() int64 { return mth.prependSize } - -func (mth *MTHFat) Reset() { panic("not implemented") } - -func (mth *MTHFat) Size() int { return MTHSize } +func (mth *MTHSeq) leafAdd() { + ent := MTHSeqEnt{c: mth.ctr} + mth.hasherLeaf.Sum(ent.h[:0]) + mth.hasherLeaf.Reset() + mth.hashes = append(mth.hashes, ent) + mth.ctr++ + if mth.events != nil { + mth.events <- MTHEvent{MTHEventAdd, &ent} + } +} -func (mth *MTHFat) BlockSize() int { return MTHBlockSize } +func (mth *MTHSeq) fold() { + for len(mth.hashes) >= 2 { + hlen := len(mth.hashes) + end1 := &mth.hashes[hlen-2] + end0 := &mth.hashes[hlen-1] + if end1.c%2 == 1 { + break + } + if end1.l != end0.l { + break + } + if _, err := mth.hasherNode.Write(end1.h[:]); err != nil { + panic(err) + } + if _, err := mth.hasherNode.Write(end0.h[:]); err != nil { + panic(err) + } + mth.hashes = mth.hashes[:hlen-1] + end1.l++ + end1.c /= 2 + mth.hasherNode.Sum(end1.h[:0]) + mth.hasherNode.Reset() + if mth.events != nil { + mth.events <- MTHEvent{MTHEventFold, end1} + } + } +} -func (mth *MTHFat) Write(data []byte) (int, error) { +func (mth *MTHSeq) Write(data []byte) (int, error) { if mth.finished { return 0, errors.New("already Sum()ed") } @@ -124,86 +207,121 @@ n, err := mth.buf.Write(data) if err != nil { return n, err } - if mth.skip > 0 && int64(mth.buf.Len()) >= mth.skip { - mth.buf.Next(int(mth.skip)) - mth.skip = 0 + if mth.toSkip > 0 { + if int64(mth.buf.Len()) < mth.toSkip { + return n, err + } + mth.buf.Next(int(mth.toSkip)) + mth.toSkip = 0 } for mth.buf.Len() >= MTHBlockSize { - if _, err = mth.hasher.Write(mth.buf.Next(MTHBlockSize)); err != nil { + if _, err = mth.hasherLeaf.Write(mth.buf.Next(MTHBlockSize)); err != nil { return n, err } - h := new([MTHSize]byte) - mth.hasher.Sum(h[:0]) - mth.hasher.Reset() - mth.hashes = append(mth.hashes, *h) - if mth.events != nil { - mth.events <- MTHEvent{ - MTHEventAppend, - 0, int64(len(mth.hashes) - 1), - mth.hashes[len(mth.hashes)-1][:], - } - } + mth.leafAdd() + mth.fold() } return n, err } -func (mth *MTHFat) PrependFrom(r io.Reader) (int64, error) { +func (mth *MTHSeq) Sum(b []byte) []byte { if mth.finished { - return 0, errors.New("already Sum()ed") + return append(b, mth.hashes[0].h[:]...) } - var err error - buf := make([]byte, MTHBlockSize) - var n int - var i, read int64 - fullsize := mth.prependSize - les := LEs{{"Pkt", mth.pktName}, {"FullSize", fullsize}, {"Size", 0}} - for mth.prependSize >= MTHBlockSize { - n, err = io.ReadFull(r, buf) - read += int64(n) - mth.prependSize -= MTHBlockSize - if err != nil { - return read, err + if mth.buf.Len() > 0 { + if _, err := mth.hasherLeaf.Write(mth.buf.Next(MTHBlockSize)); err != nil { + panic(err) } - if _, err = mth.hasher.Write(buf); err != nil { + mth.leafAdd() + mth.fold() + } + switch mth.ctr { + case 0: + if _, err := mth.hasherLeaf.Write(nil); err != nil { panic(err) } - mth.hasher.Sum(mth.hashes[i][:0]) - mth.hasher.Reset() + mth.leafAdd() + fallthrough + case 1: + ent := MTHSeqEnt{c: 1} + copy(ent.h[:], mth.hashes[0].h[:]) + mth.ctr = 2 + mth.hashes = append(mth.hashes, ent) if mth.events != nil { - mth.events <- MTHEvent{MTHEventPrepend, 0, i, mth.hashes[i][:]} + mth.events <- MTHEvent{MTHEventAdd, &ent} } - if mth.pktName != "" { - les[len(les)-1].V = read - Progress("check", les) + mth.fold() + } + for len(mth.hashes) >= 2 { + hlen := len(mth.hashes) + end1 := &mth.hashes[hlen-2] + end0 := &mth.hashes[hlen-1] + end0.l = end1.l + end0.c = end1.c + 1 + if mth.events != nil { + mth.events <- MTHEvent{MTHEventAdd, end0} } - i++ + mth.fold() + } + mth.finished = true + if mth.events != nil { + close(mth.events) + } + return append(b, mth.hashes[0].h[:]...) +} + +func MTHNew(size, offset int64) MTH { + return MTHSeqNew(size, offset) +} + +// Some kind of reference implementation (fat, because eats memory) + +type MTHFat struct { + hasher *blake3.Hasher + hashes [][MTHSize]byte + buf *bytes.Buffer + events chan MTHEvent +} + +func MTHFatNew() *MTHFat { + return &MTHFat{ + hasher: blake3.New(MTHSize, MTHLeafKey[:]), + buf: bytes.NewBuffer(make([]byte, 0, 2*MTHBlockSize)), } - if mth.prependSize > 0 { - n, err = io.ReadFull(r, buf[:mth.prependSize]) - read += int64(n) - if err != nil { - return read, err - } - if _, err = mth.hasher.Write(buf[:mth.prependSize]); err != nil { - panic(err) +} + +func (mth *MTHFat) Events() chan MTHEvent { + mth.events = make(chan MTHEvent) + return mth.events +} + +func (mth *MTHFat) Write(data []byte) (int, error) { + n, err := mth.buf.Write(data) + if err != nil { + return n, err + } + for mth.buf.Len() >= MTHBlockSize { + if _, err = mth.hasher.Write(mth.buf.Next(MTHBlockSize)); err != nil { + return n, err } - mth.hasher.Sum(mth.hashes[i][:0]) + h := new([MTHSize]byte) + mth.hasher.Sum(h[:0]) mth.hasher.Reset() + mth.hashes = append(mth.hashes, *h) if mth.events != nil { - mth.events <- MTHEvent{MTHEventPrepend, 0, i, mth.hashes[i][:]} - } - if mth.pktName != "" { - les[len(les)-1].V = fullsize - Progress("check", les) + mth.events <- MTHEvent{ + MTHEventAdd, + &MTHSeqEnt{ + 0, int64(len(mth.hashes) - 1), + mth.hashes[len(mth.hashes)-1], + }, + } } } - return read, nil + return n, err } func (mth *MTHFat) Sum(b []byte) []byte { - if mth.finished { - return append(b, mth.hashes[0][:]...) - } if mth.buf.Len() > 0 { b := mth.buf.Next(MTHBlockSize) if _, err := mth.hasher.Write(b); err != nil { @@ -215,9 +333,11 @@ mth.hasher.Reset() mth.hashes = append(mth.hashes, *h) if mth.events != nil { mth.events <- MTHEvent{ - MTHEventAppend, - 0, int64(len(mth.hashes) - 1), - mth.hashes[len(mth.hashes)-1][:], + MTHEventAdd, + &MTHSeqEnt{ + 0, int64(len(mth.hashes) - 1), + mth.hashes[len(mth.hashes)-1], + }, } } } @@ -231,17 +351,17 @@ mth.hasher.Sum(h[:0]) mth.hasher.Reset() mth.hashes = append(mth.hashes, *h) if mth.events != nil { - mth.events <- MTHEvent{MTHEventAppend, 0, 0, mth.hashes[0][:]} + mth.events <- MTHEvent{MTHEventAdd, &MTHSeqEnt{0, 0, mth.hashes[0]}} } fallthrough case 1: mth.hashes = append(mth.hashes, mth.hashes[0]) if mth.events != nil { - mth.events <- MTHEvent{MTHEventAppend, 0, 1, mth.hashes[1][:]} + mth.events <- MTHEvent{MTHEventAdd, &MTHSeqEnt{0, 1, mth.hashes[1]}} } } mth.hasher = blake3.New(MTHSize, MTHNodeKey[:]) - level := int64(1) + level := 1 for len(mth.hashes) != 1 { hashesUp := make([][MTHSize]byte, 0, 1+len(mth.hashes)/2) pairs := (len(mth.hashes) / 2) * 2 @@ -259,8 +379,10 @@ hashesUp = append(hashesUp, *h) if mth.events != nil { mth.events <- MTHEvent{ MTHEventFold, - level, int64(len(hashesUp) - 1), - hashesUp[len(hashesUp)-1][:], + &MTHSeqEnt{ + level, int64(len(hashesUp) - 1), + hashesUp[len(hashesUp)-1], + }, } } } @@ -268,180 +390,19 @@ if len(mth.hashes)%2 == 1 { hashesUp = append(hashesUp, mth.hashes[len(mth.hashes)-1]) if mth.events != nil { mth.events <- MTHEvent{ - MTHEventAppend, - level, int64(len(hashesUp) - 1), - hashesUp[len(hashesUp)-1][:], + MTHEventAdd, + &MTHSeqEnt{ + level, int64(len(hashesUp) - 1), + hashesUp[len(hashesUp)-1], + }, } } } mth.hashes = hashesUp level++ } - mth.finished = true if mth.events != nil { close(mth.events) } return append(b, mth.hashes[0][:]...) } - -type MTHSeqEnt struct { - l int64 - h [MTHSize]byte -} - -type MTHSeq struct { - hasherLeaf *blake3.Hasher - hasherNode *blake3.Hasher - hashes []MTHSeqEnt - buf *bytes.Buffer - events chan MTHEvent - ctrs []int64 - finished bool -} - -func MTHSeqNew() *MTHSeq { - mth := MTHSeq{ - hasherLeaf: blake3.New(MTHSize, MTHLeafKey[:]), - hasherNode: blake3.New(MTHSize, MTHNodeKey[:]), - buf: bytes.NewBuffer(make([]byte, 0, 2*MTHBlockSize)), - ctrs: make([]int64, 1, 2), - } - return &mth -} - -func (mth *MTHSeq) Reset() { panic("not implemented") } - -func (mth *MTHSeq) Size() int { return MTHSize } - -func (mth *MTHSeq) BlockSize() int { return MTHBlockSize } - -func (mth *MTHSeq) PrependFrom(r io.Reader) (int64, error) { - panic("must not reach that code") -} - -func (mth *MTHSeq) Events() chan MTHEvent { - mth.events = make(chan MTHEvent) - return mth.events -} - -func (mth *MTHSeq) SetPktName(pktName string) {} - -func (mth *MTHSeq) PrependSize() int64 { return 0 } - -func (mth *MTHSeq) leafAdd() { - ent := MTHSeqEnt{l: 0} - mth.hasherLeaf.Sum(ent.h[:0]) - mth.hasherLeaf.Reset() - mth.hashes = append(mth.hashes, ent) - if mth.events != nil { - mth.events <- MTHEvent{ - MTHEventAppend, 0, mth.ctrs[0], - mth.hashes[len(mth.hashes)-1].h[:], - } - } - mth.ctrs[0]++ -} - -func (mth *MTHSeq) incr(l int64) { - if int64(len(mth.ctrs)) <= l { - mth.ctrs = append(mth.ctrs, 0) - } else { - mth.ctrs[l]++ - } -} - -func (mth *MTHSeq) fold() { - for len(mth.hashes) >= 2 { - if mth.hashes[len(mth.hashes)-2].l != mth.hashes[len(mth.hashes)-1].l { - break - } - if _, err := mth.hasherNode.Write(mth.hashes[len(mth.hashes)-2].h[:]); err != nil { - panic(err) - } - if _, err := mth.hasherNode.Write(mth.hashes[len(mth.hashes)-1].h[:]); err != nil { - panic(err) - } - mth.hashes = mth.hashes[:len(mth.hashes)-1] - end := &mth.hashes[len(mth.hashes)-1] - end.l++ - mth.incr(end.l) - mth.hasherNode.Sum(end.h[:0]) - mth.hasherNode.Reset() - if mth.events != nil { - mth.events <- MTHEvent{MTHEventFold, end.l, mth.ctrs[end.l], end.h[:]} - } - } -} - -func (mth *MTHSeq) Write(data []byte) (int, error) { - if mth.finished { - return 0, errors.New("already Sum()ed") - } - n, err := mth.buf.Write(data) - if err != nil { - return n, err - } - for mth.buf.Len() >= MTHBlockSize { - if _, err = mth.hasherLeaf.Write(mth.buf.Next(MTHBlockSize)); err != nil { - return n, err - } - mth.leafAdd() - mth.fold() - } - return n, err -} - -func (mth *MTHSeq) Sum(b []byte) []byte { - if mth.finished { - return append(b, mth.hashes[0].h[:]...) - } - if mth.buf.Len() > 0 { - if _, err := mth.hasherLeaf.Write(mth.buf.Next(MTHBlockSize)); err != nil { - panic(err) - } - mth.leafAdd() - mth.fold() - } - switch mth.ctrs[0] { - case 0: - if _, err := mth.hasherLeaf.Write(nil); err != nil { - panic(err) - } - mth.leafAdd() - fallthrough - case 1: - mth.hashes = append(mth.hashes, mth.hashes[0]) - mth.ctrs[0]++ - if mth.events != nil { - mth.events <- MTHEvent{ - MTHEventAppend, 0, mth.ctrs[0], - mth.hashes[len(mth.hashes)-1].h[:], - } - } - mth.fold() - } - for len(mth.hashes) >= 2 { - l := mth.hashes[len(mth.hashes)-2].l - mth.incr(l) - mth.hashes[len(mth.hashes)-1].l = l - if mth.events != nil { - mth.events <- MTHEvent{ - MTHEventAppend, l, mth.ctrs[l], - mth.hashes[len(mth.hashes)-1].h[:], - } - } - mth.fold() - } - mth.finished = true - if mth.events != nil { - close(mth.events) - } - return append(b, mth.hashes[0].h[:]...) -} - -func MTHNew(size, offset int64) MTH { - if offset == 0 { - return MTHSeqNew() - } - return MTHFatNew(size, offset) -} diff --git a/src/mth_test.go b/src/mth_test.go index ee9b58ada83ccb240e68c34435ec2ae53dc85b50fc4d8f11b472e64d7c02431d..2e1cfe8cee6cfa71f9382cc12bda0ff2c9c8775e5f816a5abde63c62b0034268 100644 --- a/src/mth_test.go +++ b/src/mth_test.go @@ -26,7 +26,7 @@ "lukechampine.com/blake3" ) -func TestMTHFatSymmetric(t *testing.T) { +func TestMTHSeqSymmetric(t *testing.T) { xof := blake3.New(32, nil).XOF() f := func(size uint32, offset uint32) bool { size %= 2 * 1024 * 1024 @@ -36,31 +36,31 @@ panic(err) } offset = offset % size - mth := MTHFatNew(int64(size), 0) + mth := MTHSeqNew(int64(size), 0) if _, err := io.Copy(mth, bytes.NewReader(data)); err != nil { panic(err) } hsh0 := mth.Sum(nil) - mth = MTHFatNew(int64(size), int64(offset)) + mth = MTHSeqNew(int64(size), int64(offset)) if _, err := io.Copy(mth, bytes.NewReader(data[int(offset):])); err != nil { panic(err) } - if _, err := mth.PrependFrom(bytes.NewReader(data)); err != nil { + if _, err := mth.PreaddFrom(bytes.NewReader(data), "", false); err != nil { panic(err) } if bytes.Compare(hsh0, mth.Sum(nil)) != 0 { return false } - mth = MTHFatNew(0, 0) + mth = MTHSeqNew(0, 0) mth.Write(data) if bytes.Compare(hsh0, mth.Sum(nil)) != 0 { return false } data = append(data, 0) - mth = MTHFatNew(int64(size)+1, 0) + mth = MTHSeqNew(int64(size)+1, 0) if _, err := io.Copy(mth, bytes.NewReader(data)); err != nil { panic(err) } @@ -69,18 +69,18 @@ if bytes.Compare(hsh0, hsh00) == 0 { return false } - mth = MTHFatNew(int64(size)+1, int64(offset)) + mth = MTHSeqNew(int64(size)+1, int64(offset)) if _, err := io.Copy(mth, bytes.NewReader(data[int(offset):])); err != nil { panic(err) } - if _, err := mth.PrependFrom(bytes.NewReader(data)); err != nil { + if _, err := mth.PreaddFrom(bytes.NewReader(data), "", false); err != nil { panic(err) } if bytes.Compare(hsh00, mth.Sum(nil)) != 0 { return false } - mth = MTHFatNew(0, 0) + mth = MTHSeqNew(0, 0) mth.Write(data) if bytes.Compare(hsh00, mth.Sum(nil)) != 0 { return false @@ -101,12 +101,12 @@ data := make([]byte, int(size), int(size)+1) if _, err := io.ReadFull(xof, data); err != nil { panic(err) } - fat := MTHFatNew(int64(size), 0) + fat := MTHFatNew() if _, err := io.Copy(fat, bytes.NewReader(data)); err != nil { panic(err) } hshFat := fat.Sum(nil) - seq := MTHSeqNew() + seq := MTHSeqNew(int64(size), 0) if _, err := io.Copy(seq, bytes.NewReader(data)); err != nil { panic(err) } @@ -118,13 +118,13 @@ } } func TestMTHNull(t *testing.T) { - fat := MTHFatNew(0, 0) + fat := MTHFatNew() if _, err := fat.Write(nil); err != nil { t.Error(err) } hshFat := fat.Sum(nil) - seq := MTHSeqNew() + seq := MTHSeqNew(0, 0) if _, err := seq.Write(nil); err != nil { t.Error(err) } diff --git a/src/nncp.go b/src/nncp.go index 216bdcda277cf1d950dcc6c1c390bd43b03239bdc994ba65d206e18d392db9f1..e3cd31cec071c2921f035c50d652a9464133d60b8f554115027ade6d81a661e7 100644 --- a/src/nncp.go +++ b/src/nncp.go @@ -40,7 +40,7 @@ const Base32Encoded32Len = 52 var ( - Version string = "7.3.0" + Version string = "7.3.1" Base32Codec *base32.Encoding = base32.StdEncoding.WithPadding(base32.NoPadding) ) diff --git a/src/sp.go b/src/sp.go index 20744521b94d89ccc799a61fe76272bfe24f5d67275dc73023e87c92921d7515..e6be9747b404456260416ca43470bb292f020863ea43f9cde31420c457e12d2e 100644 --- a/src/sp.go +++ b/src/sp.go @@ -952,22 +952,25 @@ if state.Ctx.ShowPrgrs { state.progressBars[pktName] = struct{}{} Progress("Tx", lesp) } + if ourSize == uint64(fullSize) { + state.closeFd(pth) + state.Ctx.LogD("sp-file-finished", lesp, func(les LEs) string { + return logMsg(les) + ": finished" + }) + if state.Ctx.ShowPrgrs { + delete(state.progressBars, pktName) + } + } state.Lock() for i, q := range state.queueTheir { if *q.freq.Hash != *freq.Hash { continue } if ourSize == uint64(fullSize) { - state.Ctx.LogD("sp-file-finished", lesp, func(les LEs) string { - return logMsg(les) + ": finished" - }) state.queueTheir = append( state.queueTheir[:i], state.queueTheir[i+1:]..., ) - if state.Ctx.ShowPrgrs { - delete(state.progressBars, pktName) - } } else { q.freq.Offset = ourSize } @@ -1444,7 +1447,7 @@ continue } if hasherAndOffset != nil { delete(state.fileHashers, filePath) - if hasherAndOffset.mth.PrependSize() == 0 { + if hasherAndOffset.mth.PreaddSize() == 0 { if bytes.Compare(hasherAndOffset.mth.Sum(nil), file.Hash[:]) != 0 { state.Ctx.LogE( "sp-file-bad-checksum", lesp,