doc/.well-known/openpgpkey/hu/.gitignore | 1 - doc/.well-known/openpgpkey/hu/i4cdqgcarfjdjnba6y4jnf498asg8c6p.asc => doc/.well-known/openpgpkey/nncpgo.org/hu/i4cdqgcarfjdjnba6y4jnf498asg8c6p.asc | 0 doc/admin.texi | 4 ++-- doc/building.texi | 6 ++++++ doc/call.texi | 8 ++++++-- doc/cfg/general.texi | 2 +- doc/cmd/nncp-call.texi | 2 +- doc/cmd/nncp-rm.texi | 9 ++++----- doc/cmd/nncp-toss.texi | 2 +- doc/download.texi | 5 +++++ doc/integrity.texi | 2 +- doc/makeinfo.rc | 21 +++++++++++++++++++++ doc/multicast.texi | 16 ++++++++-------- doc/news.ru.texi | 34 ++++++++++++++++++++++++++++++++++ doc/news.texi | 34 ++++++++++++++++++++++++++++++++++ doc/nncp.html.do | 4 ++-- doc/nncp.info.do | 23 ++--------------------- doc/sp.texi | 2 +- doc/spool.texi | 14 +++++++------- makedist.sh | 4 ++-- src/cmd/nncp-bundle/main.go | 8 +++++--- src/cmd/nncp-call/main.go | 2 +- src/cmd/nncp-caller/main.go | 2 +- src/cmd/nncp-cfgnew/main.go | 2 +- src/cmd/nncp-daemon/main.go | 2 +- src/cmd/nncp-rm/main.go | 92 +++++++++++++++++++++++++++++++++++++++++------------ src/cmd/nncp-toss/main.go | 68 ++++++++++++++++++++++++++++++++++++++++-------------- src/cmd/nncp-xfer/main.go | 6 ++++-- src/ctx.go | 29 +++++++++++++++++++---------- src/df.go | 1 + src/df_netbsd.go | 1 + src/dirwatch.go | 92 +++++++++++++++++++++++++++++++++++++++++++++++++++++ src/dirwatch_dummy.go | 46 ++++++++++++++++++++++++++++++++++++++++++++++ src/go.mod | 3 ++- src/go.sum | 5 ++++- src/jobs.go | 16 +++++++++++++--- src/log.go | 8 +++++++- src/nncp.go | 2 +- src/sp.go | 21 ++++++++++++++++----- src/tmp.go | 4 ++-- src/toss.go | 63 ++++++++++++++++++++++++++++++++++++----------------- src/tx.go | 4 ++-- diff --git a/doc/.well-known/openpgpkey/hu/.gitignore b/doc/.well-known/openpgpkey/hu/.gitignore deleted file mode 100644 index 223df22bf94dbc6434a2789e9366bb7be399587333b0d4ad419592ee93ab2d28..0000000000000000000000000000000000000000 --- a/doc/.well-known/openpgpkey/hu/.gitignore +++ /dev/null @@ -1 +0,0 @@ -i4cdqgcarfjdjnba6y4jnf498asg8c6p diff --git a/doc/.well-known/openpgpkey/hu/i4cdqgcarfjdjnba6y4jnf498asg8c6p.asc b/doc/.well-known/openpgpkey/nncpgo.org/hu/i4cdqgcarfjdjnba6y4jnf498asg8c6p.asc rename from doc/.well-known/openpgpkey/hu/i4cdqgcarfjdjnba6y4jnf498asg8c6p.asc rename to doc/.well-known/openpgpkey/nncpgo.org/hu/i4cdqgcarfjdjnba6y4jnf498asg8c6p.asc diff --git a/doc/admin.texi b/doc/admin.texi index d9b348e98f3fcae030892ce35cb745d2d9492dc7549cc2cc80960c155cbd74a0..6da2fada2199ea9741c063ebfdff537c6792d897a46f27941bb6e7d90fec7ad1 100644 --- a/doc/admin.texi +++ b/doc/admin.texi @@ -24,8 +24,8 @@ It can contain @file{.nock} files: fully downloaded, but still not checksummed. Can be checksummed (with @file{.nock} extension removing) with @command{nncp-check -nock}. - Also it can contain @file{.seen} files, that should be cleaned too - from time to time. + Also it can contain @file{seen/} and @file{hdr/} subdirectories, + that should be cleaned too from time to time. All of that cleaning tasks can be done with @ref{nncp-rm} utility. diff --git a/doc/building.texi b/doc/building.texi index e54dd668e8d5cd572e80e4684c0de8bf6dc15fbf3e7841c3399d9ccfd00138d7..926722d0afdb80d70b6a899e7fe5a00dc74d6fdde48e26e019c43823cd4ccc02 100644 --- a/doc/building.texi +++ b/doc/building.texi @@ -40,3 +40,9 @@ @example # PREFIX=/usr/local redo install @end example + +NNCP depends on @code{github.com/fsnotify/fsnotify} library, that is +solely relies on OS-specific mechanisms. There is possibility that you +have either broken or unsupported ones. You can still build NNCP with +@code{-tags nofsnotify} build option, to skip @code{fsnotify} library +usage at all. diff --git a/doc/call.texi b/doc/call.texi index b27c61422429bcdbc971a621c3c38d24d16204d8c56c16ef713c589efdd7d61d..e2e3d241106f645fd0ab7192fcf25b03efac05738a1425e0ad0e1a41cdbfe776 100644 --- a/doc/call.texi +++ b/doc/call.texi @@ -81,11 +81,15 @@ configuration option when calling. @item autotoss, -doseen, -nofile, -nofreq, -noexec, -notrns Optionally enable auto tossing: run tosser on node's spool every second -during the call. You can control either are @file{.seen} files must be +during the call. You can control either are @file{seen/} files must be created, or skip any kind of packet processing. @item when-tx-exists -Call only if packets for sending exists. +Call only if packets for sending exists. The check of outbound packets +existence is performed @strong{every} time we are going to make a call, +but @emph{when-tx-exists} does not influence @emph{cron}. When +@emph{cron} configuration decides that it is time to make a call, with +@emph{when-tx-exists} option it checks packets existence first. @anchor{CfgNoCK} @item nock diff --git a/doc/cfg/general.texi b/doc/cfg/general.texi index 85142d33759ebccd99425c975ed55cea1a154ca3b56cbc04da825fc9f230113c..9a86d436284a95f1f9ee271a12242957ce9711afc770989e44b729b8038b17b9 100644 --- a/doc/cfg/general.texi +++ b/doc/cfg/general.texi @@ -36,7 +36,7 @@ You can always force its showing with @option{-progress} command line option anyway. @anchor{CfgNoHdr} @item nohdr -@strong{nohdr} option disables @ref{HdrFile, .hdr} files usage. +@strong{nohdr} option disables @ref{HdrFile, @file{hdr/}} files usage. @end table And optional @ref{MCD, MultiCast Discovery} options: diff --git a/doc/cmd/nncp-call.texi b/doc/cmd/nncp-call.texi index ab1c05fa1ed8d2f1fa11f2dbde3022a6e75c88a2f3e0a002d2d7ba036e5e3dd1..bd664a307fc494ebb5a2c7a599d99acc4b7dbdfc6cbb670c490e2f3374833cdc 100644 --- a/doc/cmd/nncp-call.texi +++ b/doc/cmd/nncp-call.texi @@ -61,5 +61,5 @@ node won't be notified that the file is finished. If you run @ref{nncp-check, @command{nncp-check -nock}}, that will checksum files and strip the @file{.nock} extension, then repeated call to remote node will notify about packet's completion. Also it will be notified if -@ref{nncp-toss, tossing} created @file{.seen} file. +@ref{nncp-toss, tossing} created @file{seen/} file. Read @ref{CfgNoCK, more} about @option{-nock} option. diff --git a/doc/cmd/nncp-rm.texi b/doc/cmd/nncp-rm.texi index 3fcad58443e8fa7b63bf2e03547731c81d0d3748295bd141dd5a85cf5b62ad63..6f433ec08dbd92363bd3c508d8cd283cbe8a4be83a562f4598a247e71bba526b 100644 --- a/doc/cmd/nncp-rm.texi +++ b/doc/cmd/nncp-rm.texi @@ -33,16 +33,15 @@ (maybe both of them), then delete all packets from that given queues. @item @option{-part} option deletes @file{.part}ly downloaded files. -@item @option{-seen} option deletes @file{.seen} files. But it does not -apply to @ref{Multicast, multicast areas} @file{.seen} ones! +@item @option{-seen} option deletes @file{seen/} files. But it does not +apply to @ref{Multicast, multicast areas} ones! @item @option{-nock} option deletes non-checksummed (non-verified) @file{.nock} files. -@item @option{-hdr} option deletes cached @file{.hdr} files. +@item @option{-hdr} option deletes cached @file{hdr/} files. -@item @option{-area} option deletes @file{.seen} files in @file{area/} -subdirectories. +@item @option{-area} option deletes seen files in @file{area/} subdirectories. @end itemize diff --git a/doc/cmd/nncp-toss.texi b/doc/cmd/nncp-toss.texi index 2d359a4f88d33a26a111b45b9f9cf303a6fc04e7463e92078887dbc66179566a..f5550fc993b01dc6da0a0a6143fc1598bba83db96fa3cd48e83e54dca2bcd1cc 100644 --- a/doc/cmd/nncp-toss.texi +++ b/doc/cmd/nncp-toss.texi @@ -22,7 +22,7 @@ @option{-cycle} option tells not to quit, but to repeat tossing every @option{INT} seconds in an infinite loop. That can be useful when running this command as a daemon. -@option{-seen} option creates empty @file{XXX.seen} file after +@option{-seen} option creates empty @file{seen/XXX} file after successful tossing of @file{XXX} packet. @ref{nncp-xfer}, @ref{nncp-bundle}, @ref{nncp-daemon} and @ref{nncp-call} commands skip inbound packets that has been already seen, processed and tossed. This diff --git a/doc/download.texi b/doc/download.texi index 77aac8ce3617f90dda6423666805835637e9717e9a49897673bf85e03f2f1ac9..4f15d6a952db6421c7be38a70b98f165897d5fd9c321f776d19f43ee75b9d7cc 100644 --- a/doc/download.texi +++ b/doc/download.texi @@ -12,6 +12,7 @@ @headitem Library @tab Licence @item @code{github.com/davecgh/go-xdr} @tab ISC @item @code{github.com/dustin/go-humanize} @tab MIT @item @code{github.com/flynn/noise} @tab BSD 3-Clause +@item @code{github.com/fsnotify/fsnotify} @tab BSD 3-Clause @item @code{github.com/gorhill/cronexpr} @tab GNU GPLv3 @item @code{github.com/gosuri/uilive} @tab MIT @item @code{github.com/hjson/hjson-go} @tab MIT @@ -28,6 +29,10 @@ @end multitable @multitable {XXXXX} {XXXX-XX-XX} {XXXX KiB} {link sign} {xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx} @headitem Version @tab Date @tab Size @tab Tarball @tab SHA256 checksum + +@item @ref{Release 7_6_0, 7.6.0} @tab 2021-08-08 @tab 1153 KiB +@tab @url{download/nncp-7.6.0.tar.xz, link} @url{download/nncp-7.6.0.tar.xz.sig, sign} +@tab @code{00852E80 70415154 197A5555 DDAE636E 6E3940EC DD53D39E A69E5FF1 531BA4C6} @item @ref{Release 7_5_1, 7.5.1} @tab 2021-08-05 @tab 1147 KiB @tab @url{download/nncp-7.5.1.tar.xz, link} @url{download/nncp-7.5.1.tar.xz.sig, sign} diff --git a/doc/integrity.texi b/doc/integrity.texi index d2e33f0efef8ab90c514ab68d9a487d58b9edf38a72914ab87d18f04768a6f04..a0884eb9184d356929686b3ae53c052708220ee4ce6d7c5cbeed750daa979ccd 100644 --- a/doc/integrity.texi +++ b/doc/integrity.texi @@ -25,7 +25,7 @@ $ gpg --auto-key-locate wkd --locate-keys releases at nncpgo dot org @end example @item -@verbatiminclude .well-known/openpgpkey/hu/i4cdqgcarfjdjnba6y4jnf498asg8c6p.asc +@verbatiminclude .well-known/openpgpkey/nncpgo.org/hu/i4cdqgcarfjdjnba6y4jnf498asg8c6p.asc @end itemize diff --git a/doc/makeinfo.rc b/doc/makeinfo.rc new file mode 100644 index 0000000000000000000000000000000000000000..322b1ac0ff6050f5454af9c6e65b7563a82a69958b287dec4e1367474257b850 --- /dev/null +++ b/doc/makeinfo.rc @@ -0,0 +1,21 @@ +redo-ifchange \ + ../config \ + ../VERSION \ + *.texi \ + cfg/*.texi \ + cmd/*.texi \ + integration/*.texi \ + pedro.txt \ + pkt/*.texi \ + sp.plantuml.txt \ + usecases.ru/*.texi \ + usecases/*.texi +. ../config +${MAKEINFO:-makeinfo} \ + -D "VERSION `cat ../VERSION`" \ + $MAKEINFO_OPTS \ + --set-customization-variable SECTION_NAME_IN_TITLE=1 \ + --set-customization-variable TREE_TRANSFORMATIONS=complete_tree_nodes_menus \ + --set-customization-variable CLOSE_QUOTE_SYMBOL=\" \ + --set-customization-variable OPEN_QUOTE_SYMBOL=\" \ + --output $3 index.texi diff --git a/doc/multicast.texi b/doc/multicast.texi index e7e07e5b1189a5b5c5b3637d28d7e7812050b70624e2555fdec9b3dbdbbf2c29..182c8c1ea18b62afda55e07956282bd253316ace43f9f9e41d3fce36997d27d9 100644 --- a/doc/multicast.texi +++ b/doc/multicast.texi @@ -39,12 +39,12 @@ @item for each area's subscribers: @itemize @item check if that message was already seen (sent or received from) before by the destination node: check existence of - @file{SPOOL/NODE/area/AREA/MsgHash.seen} file. Skip that node if + @file{SPOOL/NODE/area/AREA/MsgHash} file. Skip that node if it exists @item if subscriber's node is not the one we received the packet from, then create outgoing encrypted packet to it, with that area packet inside - @item create corresponding @file{MsgHash.seen} file + @item create corresponding @file{MsgHash} file @item "rewind" the outer encrypted file to the beginning and repeat the whole cycle again, while all of subscribers will "seen" that area's message. @@ -55,7 +55,7 @@ cryptography will be in use, having negligible CPU resource consumption. @end itemize @item check if we have seen that area's message before by looking at - @file{SPOOL/SELF/area/AREA/MsgHash.seen}. If so, remove the packet, + @file{SPOOL/SELF/area/AREA/MsgHash}. If so, remove the packet, because it is just a ordinary possible duplicate, finish its processing @item check if we have got corresponding area's private key. If no key exists, then remove the packet, finish its processing -- we just @@ -109,15 +109,15 @@ @item @command{nncp-file} creates an encrypted packet with area packet and encrypted packet inside it, with our own @code{self} node as a recipient (in the @file{SPOOL/SELF/tx} directory). It also creates the -@file{SPOOL/SELF/area/AREA/MSGHASH.seen} file. +@file{SPOOL/SELF/area/AREA/MsgHash} file. @item @command{nncp-toss} sees @file{tx/} file and "opens" it, applying the area message tossing procedure as described above. That will create outgoing packets in @file{SPOOL/nodeB/tx} and @file{SPOOL/nodeD/tx} -directories with @file{SPOOL/nodeB/area/AREA/MSGHASH.seen} -@file{SPOOL/nodeD/area/AREA/MSGHASH.seen} files. Because we already have -@file{SPOOL/SELF/area/AREA/MSGHASH.seen}, that packet is removed then. +directories with @file{SPOOL/nodeB/area/AREA/MsgHash} +@file{SPOOL/nodeD/area/AREA/MsgHash} files. Because we already have +@file{SPOOL/SELF/area/AREA/MsgHash}, that packet is removed then. @item When @code{nodeB} receives the encrypted packet, it sees the area one @@ -131,7 +131,7 @@ @item @code{nodeD} receives packets from both @code{nodeA} and @code{nodeB}. Only one of them processed, and other is ignored because corresponding -@file{MSGHASH.seen} file will exist. +@file{MsgHash} file will exist. If @code{nodeD} will receive packet from the @code{nodeB} first, it will relay it to the @code{nodeA} also, that will silently remove it when diff --git a/doc/news.ru.texi b/doc/news.ru.texi index d6181bc66dcf4178c0d5c3eca5c650a9a7b9dfcb110e2c1b86ca87cb181b2ac7..ff3f47b474101839ebe3150512585a2ea1d408ae810628f36e02fe23c0886fe7 100644 --- a/doc/news.ru.texi +++ b/doc/news.ru.texi @@ -1,6 +1,40 @@ @node Новости @section Новости +@node Релиз 7.7.0 +@subsection Релиз 7.7.0 +@itemize + +@item +Экспериментальная поддержка @code{kqueue} и @code{inotify} оповещений об +изменениях в spool директориях, для сокращения накладных расходов на их +частое чтение. + +@item +@file{.seen} и @file{.hdr} файлы находятся в @file{seen/} и @file{hdr/} +поддиректориях теперь, дабы ускорить сканирование spool областей. +Необходима миграция текущих файлов: + +@example +$ find $NNCPSPOOL -type f -name "*.hdr" -exec rm @{@} + + +$ find $NNCPSPOOL -type d -name rx | while read rx ; do + cd $rx + mkdir -p seen + find . -type f -name "*.seen" | while read fn ; do + mv $fn seen/$@{fn%.seen@} + done +done + +$ find $NNCPSPOOL -type d -name area | while read area ; do + find $area -type f -name "*.seen" | while read fn ; do + mv $fn $@{fn%.seen@} + done +done +@end example + +@end itemize + @node Релиз 7.6.0 @subsection Релиз 7.6.0 @itemize diff --git a/doc/news.texi b/doc/news.texi index c823541db69b8744c37294d6387ea9cffb98e9679c36f9e1442a7766ceb5ef9b..d71478fc71c95306fad5e318aa4019dcb2380b24028303eea343b4f54d8281ab 100644 --- a/doc/news.texi +++ b/doc/news.texi @@ -3,6 +3,40 @@ @unnumbered News See also this page @ref{Новости, on russian}. +@node Release 7_7_0 +@section Release 7.7.0 +@itemize + +@item +Experimental @code{kqueue} and @code{inotify} based notifications +support about spool directory changes, for reducing their often reading +overhead. + +@item +@file{.seen} and @file{.hdr} files moved to @file{seen/} and @file{hdr/} +subdirectories, for faster scanning of spool directories. +Current files migration required: + +@example +$ find $NNCPSPOOL -type f -name "*.hdr" -exec rm @{@} + + +$ find $NNCPSPOOL -type d -name rx | while read rx ; do + cd $rx + mkdir -p seen + find . -type f -name "*.seen" | while read fn ; do + mv $fn seen/$@{fn%.seen@} + done +done + +$ find $NNCPSPOOL -type d -name area | while read area ; do + find $area -type f -name "*.seen" | while read fn ; do + mv $fn $@{fn%.seen@} + done +done +@end example + +@end itemize + @node Release 7_6_0 @section Release 7.6.0 @itemize diff --git a/doc/nncp.html.do b/doc/nncp.html.do index cc247f5881c5008cb58c356518813cea89f0f7aa6a8d39cc653ce4d2ac8b74f7..40a5e601567da2ab9ddc451119a1baff265ab3d0826af602fa87c2a83a08c357 100644 --- a/doc/nncp.html.do +++ b/doc/nncp.html.do @@ -1,7 +1,7 @@ +redo-ifchange makeinfo.rc rm -fr nncp.html MAKEINFO_OPTS="$MAKEINFO_OPTS --html --css-include style.css" MAKEINFO_OPTS="$MAKEINFO_OPTS --set-customization-variable FORMAT_MENU=menu" MAKEINFO_OPTS="$MAKEINFO_OPTS --set-customization-variable SHOW_TITLE=0" MAKEINFO_OPTS="$MAKEINFO_OPTS --set-customization-variable DATE_IN_HEADER=1" -MAKEINFO_OPTS="$MAKEINFO_OPTS" . nncp.info.do -cp -r .well-known $3 +MAKEINFO_OPTS="$MAKEINFO_OPTS" . makeinfo.rc diff --git a/doc/nncp.info.do b/doc/nncp.info.do index 322b1ac0ff6050f5454af9c6e65b7563a82a69958b287dec4e1367474257b850..b974dd16bbb011415a4599f9294ef4610fb6cb36f8b8a76c52985d4338323846 100644 --- a/doc/nncp.info.do +++ b/doc/nncp.info.do @@ -1,21 +1,2 @@ -redo-ifchange \ - ../config \ - ../VERSION \ - *.texi \ - cfg/*.texi \ - cmd/*.texi \ - integration/*.texi \ - pedro.txt \ - pkt/*.texi \ - sp.plantuml.txt \ - usecases.ru/*.texi \ - usecases/*.texi -. ../config -${MAKEINFO:-makeinfo} \ - -D "VERSION `cat ../VERSION`" \ - $MAKEINFO_OPTS \ - --set-customization-variable SECTION_NAME_IN_TITLE=1 \ - --set-customization-variable TREE_TRANSFORMATIONS=complete_tree_nodes_menus \ - --set-customization-variable CLOSE_QUOTE_SYMBOL=\" \ - --set-customization-variable OPEN_QUOTE_SYMBOL=\" \ - --output $3 index.texi +redo-ifchange makeinfo.rc +MAKEINFO_OPTS="--no-split" . makeinfo.rc diff --git a/doc/sp.texi b/doc/sp.texi index 2c5c7337c01c7fbf9a20652681119f9e2136f9272f37532c3bfe8247b8925a05..7d170f8c9ee3295734c1561f2209b980c58261c6f596ee360a2d8617a84d3add 100644 --- a/doc/sp.texi +++ b/doc/sp.texi @@ -182,7 +182,7 @@ @item Check that it has an acceptable niceness level. Ignore it if it is too nice. @item If already downloaded file exists, then queue @emph{DONE} sending. - @item If @file{.seen} exists, then queue @emph{DONE} sending. + @item If @file{seen/XXX} exists, then queue @emph{DONE} sending. @item If @file{.part} exists, then queue @emph{FREQ} sending with corresponding offset. @end itemize diff --git a/doc/spool.texi b/doc/spool.texi index c2b23d073bbad95e43cad8948dce5f9c99f745e0cb7fe9bf677b301babe00cc7..78964071cddec16935cb31ec5a0f2fa77c93ef6f3795fb046d1aded4c56f9b2d 100644 --- a/doc/spool.texi +++ b/doc/spool.texi @@ -50,9 +50,9 @@ non-checksummed (NoCK) @strong{fully} received file. Its checksum is verified against its filename either by @ref{nncp-check}, or by working online daemons. If it is correct, then its extension is trimmed. -@item LYT64MWSNDK34CVYOO7TA6ZCJ3NWI2OUDBBMX2A4QWF34FIRY4DQ.seen +@item seen/LYT64MWSNDK34CVYOO7TA6ZCJ3NWI2OUDBBMX2A4QWF34FIRY4DQ @ref{nncp-toss} utility can be invoked with @option{-seen} option, -leading to creation of @file{.seen} files, telling that the file with +leading to creation of @file{seen/} files, telling that the file with specified hash has already been processed before. It could be useful when there are use-cases where multiple ways of packets transfer available and there is possibility of duplicates reception. You have to @@ -60,9 +60,9 @@ manually remove them, when you do not need them (probably because they are expired). @anchor{HdrFile} -@item LYT64MWSNDK34CVYOO7TA6ZCJ3NWI2OUDBBMX2A4QWF34FIRY4DQ.hdr +@item hdr/LYT64MWSNDK34CVYOO7TA6ZCJ3NWI2OUDBBMX2A4QWF34FIRY4DQ If no @ref{CfgNoHdr, nohdr} option is enabled in configuration file, -then @file{.hdr} files are automatically created for every ordinary +then @file{hdr/} files are automatically created for every ordinary (fully received and checksummed) packet. It literally contains just the header of the corresponding packet. It will be automatically created even during simple @ref{nncp-stat} call. On filesystems with big @@ -71,9 +71,9 @@ directories, because it prevents unnecessary read-amplification. On other filesystems probably it won't help at all, or even harm performance. -There is a hack: you can create more dense @file{.hdr} allocation by -removing all @file{.hdr} files and then running @command{nncp-stat}, -that will recreate them. In many cases many @file{.hdr} files will be +There is a hack: you can create more dense @file{hdr/} allocation by +removing all @file{hdr/} files and then running @command{nncp-stat}, +that will recreate them. In many cases many @file{hdr/} files will be allocated more or less linearly on the disk, decreasing listing time even more. diff --git a/makedist.sh b/makedist.sh index 3901c5ededb0ecf95fb165b0de0c843c4592e6bb1756e2d60cd4ee8b99571385..91880935f82e7f114a87de3be8a5e5553d65986fa1dee71e542a72f12989d479 100755 --- a/makedist.sh +++ b/makedist.sh @@ -38,7 +38,7 @@ @url{http://www.nncpgo.org/Mirrors.html, mirrors}. EOF perl -i -ne 'print unless /include pedro/' doc/index.texi doc/about.ru.texi perl -p -i -e 's/^(.verbatiminclude) .*$/$1 PUBKEY.asc/g' doc/integrity.texi -mv doc/.well-known/openpgpkey/hu/i4cdqgcarfjdjnba6y4jnf498asg8c6p.asc PUBKEY.asc +mv doc/.well-known/openpgpkey/nncpgo.org/hu/i4cdqgcarfjdjnba6y4jnf498asg8c6p.asc PUBKEY.asc ln -s ../PUBKEY.asc doc redo doc @@ -95,7 +95,7 @@ EOF mkinfo --output THANKS $texi rm -f $texi -rm -r doc/.well-known doc/nncp.html/.well-known +rm -r doc/.well-known ######################################################################## diff --git a/src/cmd/nncp-bundle/main.go b/src/cmd/nncp-bundle/main.go index 4f8e366f2556eabaf27205d2744489bfae564a91c9575ea8ac77666b5061ba57..364088c8d886453656dac39f1d3ae8cef1356ef2af9e520b403866bd1028362e 100644 --- a/src/cmd/nncp-bundle/main.go +++ b/src/cmd/nncp-bundle/main.go @@ -187,7 +187,7 @@ if *doDelete { if err = os.Remove(job.Path); err != nil { log.Fatalln("Error during deletion:", err) } else if ctx.HdrUsage { - os.Remove(job.Path + nncp.HdrSuffix) + os.Remove(nncp.JobPath2Hdr(job.Path)) } } ctx.LogI( @@ -376,7 +376,7 @@ }) if !*dryRun { os.Remove(dstPath) if ctx.HdrUsage { - os.Remove(dstPath + nncp.HdrSuffix) + os.Remove(nncp.JobPath2Hdr(dstPath)) } } } else { @@ -416,7 +416,9 @@ return logMsg(les) + ": packet already exists" }) continue } - if _, err = os.Stat(dstPath + nncp.SeenSuffix); err == nil || !os.IsNotExist(err) { + if _, err = os.Stat(filepath.Join( + dstDirPath, nncp.SeenDir, pktName, + )); err == nil || !os.IsNotExist(err) { ctx.LogD("bundle-rx-seen", les, func(les nncp.LEs) string { return logMsg(les) + ": packet already seen" }) diff --git a/src/cmd/nncp-call/main.go b/src/cmd/nncp-call/main.go index 8e65c278c223a5c83310247e6e14995f6494fce069e9993fba6e750626acd68a..6ce8afa2731e620420b06987ca96955c3138086f353b556c78ef15672e794f91 100644 --- a/src/cmd/nncp-call/main.go +++ b/src/cmd/nncp-call/main.go @@ -62,7 +62,7 @@ onlineDeadlineSec = flag.Uint("onlinedeadline", 0, "Override onlinedeadline option") maxOnlineTimeSec = flag.Uint("maxonlinetime", 0, "Override maxonlinetime option") autoToss = flag.Bool("autotoss", false, "Toss after call is finished") - autoTossDoSeen = flag.Bool("autotoss-seen", false, "Create .seen files during tossing") + autoTossDoSeen = flag.Bool("autotoss-seen", false, "Create seen/ files during tossing") autoTossNoFile = flag.Bool("autotoss-nofile", false, "Do not process \"file\" packets during tossing") autoTossNoFreq = flag.Bool("autotoss-nofreq", false, "Do not process \"freq\" packets during tossing") autoTossNoExec = flag.Bool("autotoss-noexec", false, "Do not process \"exec\" packets during tossing") diff --git a/src/cmd/nncp-caller/main.go b/src/cmd/nncp-caller/main.go index b75b251049d9a467aa970e56ad5d235d04ef0e08d10e342120bd196fc836bd82..4cd601ead0c108df5214cba0001d15aca95e25d5e0659a111a614b8a319ab633 100644 --- a/src/cmd/nncp-caller/main.go +++ b/src/cmd/nncp-caller/main.go @@ -51,7 +51,7 @@ version = flag.Bool("version", false, "Print version information") warranty = flag.Bool("warranty", false, "Print warranty information") autoToss = flag.Bool("autotoss", false, "Toss after call is finished") - autoTossDoSeen = flag.Bool("autotoss-seen", false, "Create .seen files during tossing") + autoTossDoSeen = flag.Bool("autotoss-seen", false, "Create seen/ files during tossing") autoTossNoFile = flag.Bool("autotoss-nofile", false, "Do not process \"file\" packets during tossing") autoTossNoFreq = flag.Bool("autotoss-nofreq", false, "Do not process \"freq\" packets during tossing") autoTossNoExec = flag.Bool("autotoss-noexec", false, "Do not process \"exec\" packets during tossing") diff --git a/src/cmd/nncp-cfgnew/main.go b/src/cmd/nncp-cfgnew/main.go index 212a4d6529a74ea44d4dc6813684bbe9c0e9812fd65673ef93524eb86a7210b9..8fcbd14d4ca9280821ac35d1079313c3b3f6da93a269d7f60e998a5b493839e2 100644 --- a/src/cmd/nncp-cfgnew/main.go +++ b/src/cmd/nncp-cfgnew/main.go @@ -179,7 +179,7 @@ # Enforce specified umask usage # umask: "022" # Omit progress showing by default # noprogress: true - # Do not use .hdr files + # Do not use hdr/ files # nohdr: true # MultiCast Discovery: diff --git a/src/cmd/nncp-daemon/main.go b/src/cmd/nncp-daemon/main.go index f4fa5eb3f882a87d837a594b2efbdba773b12137a6be4ec16ce0ddd6f86509dc..8d5e1ed2f8056e11c5287233b59b966368092811bb46a8acc9f7cd98902117cb 100644 --- a/src/cmd/nncp-daemon/main.go +++ b/src/cmd/nncp-daemon/main.go @@ -124,7 +124,7 @@ version = flag.Bool("version", false, "Print version information") warranty = flag.Bool("warranty", false, "Print warranty information") autoToss = flag.Bool("autotoss", false, "Toss after call is finished") - autoTossDoSeen = flag.Bool("autotoss-seen", false, "Create .seen files during tossing") + autoTossDoSeen = flag.Bool("autotoss-seen", false, "Create seen/ files during tossing") autoTossNoFile = flag.Bool("autotoss-nofile", false, "Do not process \"file\" packets during tossing") autoTossNoFreq = flag.Bool("autotoss-nofreq", false, "Do not process \"freq\" packets during tossing") autoTossNoExec = flag.Bool("autotoss-noexec", false, "Do not process \"exec\" packets during tossing") diff --git a/src/cmd/nncp-rm/main.go b/src/cmd/nncp-rm/main.go index bfc863d4fcdb023acb4316072f95171a729c91cae84bae9931785098d7a95aed..821b353684bf181288d09cd00249537125dfa62f71d406da7ec3e01c4037364c 100644 --- a/src/cmd/nncp-rm/main.go +++ b/src/cmd/nncp-rm/main.go @@ -54,15 +54,15 @@ var ( cfgPath = flag.String("cfg", nncp.DefaultCfgPath, "Path to configuration file") doAll = flag.Bool("all", false, "Apply remove rules to all nodes") doTmp = flag.Bool("tmp", false, "Remove all temporary files") - doHdr = flag.Bool("hdr", false, "Remove all .hdr files") + doHdr = flag.Bool("hdr", false, "Remove all hdr/ files") doLock = flag.Bool("lock", false, "Remove all lock files") nodeRaw = flag.String("node", "", "Node to remove files in") doRx = flag.Bool("rx", false, "Process received packets") doTx = flag.Bool("tx", false, "Process transfered packets") doPart = flag.Bool("part", false, "Remove only .part files") - doSeen = flag.Bool("seen", false, "Remove only .seen files") + doSeen = flag.Bool("seen", false, "Remove only seen/ files") doNoCK = flag.Bool("nock", false, "Remove only .nock files") - doArea = flag.Bool("area", false, "Remove only area/*.seen files") + doArea = flag.Bool("area", false, "Remove only area/* seen files") older = flag.String("older", "", "XXX{smhd}: only older than XXX number of time units") dryRun = flag.Bool("dryrun", false, "Do not actually remove files") pktRaw = flag.String("pkt", "", "Packet to remove") @@ -204,9 +204,7 @@ return fmt.Sprintf("File %s: too fresh, skipping", path) }) return nil } - if (*doSeen && strings.HasSuffix(info.Name(), nncp.SeenSuffix)) || - (*doNoCK && strings.HasSuffix(info.Name(), nncp.NoCKSuffix)) || - (*doHdr && strings.HasSuffix(info.Name(), nncp.HdrSuffix)) || + if (*doNoCK && strings.HasSuffix(info.Name(), nncp.NoCKSuffix)) || (*doPart && strings.HasSuffix(info.Name(), nncp.PartSuffix)) { ctx.LogI("rm", nncp.LEs{{K: "File", V: path}}, logMsg) if *dryRun { @@ -233,7 +231,7 @@ } return nil }) } - if *pktRaw != "" || *doRx || *doSeen || *doNoCK || *doHdr || *doPart { + if *pktRaw != "" || *doRx || *doNoCK || *doPart { if err = remove(nncp.TRx); err != nil { log.Fatalln("Can not remove:", err) } @@ -243,6 +241,63 @@ if err = remove(nncp.TTx); err != nil { log.Fatalln("Can not remove:", err) } } + removeSub := func(p string, everything bool) error { + return filepath.Walk( + p, func(path string, info os.FileInfo, err error) error { + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + if info.IsDir() { + return nil + } + logMsg := func(les nncp.LEs) string { + return fmt.Sprintf("File %s: removed", path) + } + if everything { + ctx.LogI("rm", nncp.LEs{{K: "File", V: path}}, logMsg) + if *dryRun { + return nil + } + return os.Remove(path) + } + if now.Sub(info.ModTime()) < oldBoundary { + ctx.LogD( + "rm-skip", nncp.LEs{{K: "File", V: path}}, + func(les nncp.LEs) string { + return fmt.Sprintf("File %s: too fresh, skipping", path) + }, + ) + } else if !*dryRun { + return os.Remove(path) + } + return nil + }, + ) + } + if *doRx || *doSeen { + if err = removeSub(filepath.Join( + ctx.Spool, node.Id.String(), string(nncp.TRx), nncp.SeenDir, + ), *doSeen); err != nil { + log.Fatalln("Can not remove:", err) + } + } + if *doRx || *doHdr { + if err = removeSub(filepath.Join( + ctx.Spool, node.Id.String(), string(nncp.TRx), nncp.HdrDir, + ), *doHdr); err != nil { + log.Fatalln("Can not remove:", err) + } + } + if *doTx || *doHdr { + if err = removeSub(filepath.Join( + ctx.Spool, node.Id.String(), string(nncp.TTx), nncp.HdrDir, + ), *doHdr); err != nil { + log.Fatalln("Can not remove:", err) + } + } if *doArea { if err = filepath.Walk( filepath.Join(ctx.Spool, node.Id.String(), nncp.AreaDir), @@ -262,20 +317,17 @@ return fmt.Sprintf("File %s: too fresh, skipping", path) }) return nil } - if strings.HasSuffix(info.Name(), nncp.SeenSuffix) { - ctx.LogI( - "rm", - nncp.LEs{{K: "File", V: path}}, - func(les nncp.LEs) string { - return fmt.Sprintf("File %s: removed", path) - }, - ) - if *dryRun { - return nil - } - return os.Remove(path) + ctx.LogI( + "rm", + nncp.LEs{{K: "File", V: path}}, + func(les nncp.LEs) string { + return fmt.Sprintf("File %s: removed", path) + }, + ) + if *dryRun { + return nil } - return nil + return os.Remove(path) }); err != nil { log.Fatalln("Can not remove:", err) } diff --git a/src/cmd/nncp-toss/main.go b/src/cmd/nncp-toss/main.go index 6f62b8737f9a207e9a7094b735b2577617e6a5116ff016fd7ec6e93ca4ae4f6a..874b72aeb8007571950eb29ffa5c5c463710c83923b92b244012620557c04954 100644 --- a/src/cmd/nncp-toss/main.go +++ b/src/cmd/nncp-toss/main.go @@ -23,6 +23,7 @@ "flag" "fmt" "log" "os" + "path/filepath" "time" "go.cypherpunks.ru/nncp/v7" @@ -41,7 +42,7 @@ cfgPath = flag.String("cfg", nncp.DefaultCfgPath, "Path to configuration file") nodeRaw = flag.String("node", "", "Process only that node") niceRaw = flag.String("nice", nncp.NicenessFmt(255), "Minimal required niceness") dryRun = flag.Bool("dryrun", false, "Do not actually write any tossed data") - doSeen = flag.Bool("seen", false, "Create .seen files") + doSeen = flag.Bool("seen", false, "Create seen/ files") cycle = flag.Uint("cycle", 0, "Repeat tossing after N seconds in infinite loop") noFile = flag.Bool("nofile", false, "Do not process \"file\" packets") noFreq = flag.Bool("nofreq", false, "Do not process \"freq\" packets") @@ -99,32 +100,65 @@ } ctx.Umask() -Cycle: - isBad := false + if *cycle == 0 { + isBad := false + for nodeId, node := range ctx.Neigh { + if nodeOnly != nil && nodeId != *nodeOnly.Id { + continue + } + isBad = ctx.Toss( + node.Id, + nncp.TRx, + nice, + *dryRun, *doSeen, *noFile, *noFreq, *noExec, *noTrns, *noArea, + ) || isBad + if nodeId == *ctx.SelfId { + isBad = ctx.Toss( + node.Id, + nncp.TTx, + nice, + *dryRun, false, true, true, true, true, *noArea, + ) || isBad + } + } + if isBad { + os.Exit(1) + } + return + } + + nodeIds := make(chan *nncp.NodeId) for nodeId, node := range ctx.Neigh { if nodeOnly != nil && nodeId != *nodeOnly.Id { continue } - isBad = ctx.Toss( - node.Id, + dw, err := ctx.NewDirWatcher( + filepath.Join(ctx.Spool, node.Id.String(), string(nncp.TRx)), + time.Second*time.Duration(*cycle), + ) + if err != nil { + log.Fatalln(err) + } + go func(nodeId *nncp.NodeId) { + for range dw.C { + nodeIds <- nodeId + } + }(node.Id) + } + for nodeId := range nodeIds { + ctx.Toss( + nodeId, nncp.TRx, nice, *dryRun, *doSeen, *noFile, *noFreq, *noExec, *noTrns, *noArea, - ) || isBad - if nodeId == *ctx.SelfId { - isBad = ctx.Toss( - node.Id, + ) + if *nodeId == *ctx.SelfId { + ctx.Toss( + nodeId, nncp.TTx, nice, *dryRun, false, true, true, true, true, *noArea, - ) || isBad + ) } - } - if *cycle > 0 { - time.Sleep(time.Duration(*cycle) * time.Second) - goto Cycle - } - if isBad { - os.Exit(1) } } diff --git a/src/cmd/nncp-xfer/main.go b/src/cmd/nncp-xfer/main.go index 59b036ee65636481ec2b94324080bf4439c8195dd63ae1d573334c729bfe0d82..1437126002471fcd99634ba1d08a62793973f35f34b8c4fbd4468aed6d76b823 100644 --- a/src/cmd/nncp-xfer/main.go +++ b/src/cmd/nncp-xfer/main.go @@ -423,7 +423,9 @@ return logMsg(les) + ": already exists" }) continue } - if _, err = os.Stat(filepath.Join(dstPath, pktName+nncp.SeenSuffix)); err == nil || !os.IsNotExist(err) { + if _, err = os.Stat(filepath.Join( + dstPath, nncp.SeenDir, pktName, + )); err == nil || !os.IsNotExist(err) { ctx.LogD("xfer-tx-seen", les, func(les nncp.LEs) string { return logMsg(les) + ": already seen" }) @@ -518,7 +520,7 @@ return logMsg(les) + ": removing" }) isBad = true } else if ctx.HdrUsage { - os.Remove(job.Path + nncp.HdrSuffix) + os.Remove(nncp.JobPath2Hdr(job.Path)) } } } diff --git a/src/ctx.go b/src/ctx.go index 1777a12a865ec238667f4cbeb347b84dea020202a2391d7544501c9ac3633421..c432c58184cd04f2c9228bd42e23ccf0ff26e91589608fea5dbea66b9025ad14 100644 --- a/src/ctx.go +++ b/src/ctx.go @@ -69,21 +69,30 @@ } return node, nil } -func (ctx *Ctx) ensureRxDir(nodeId *NodeId) error { - dirPath := filepath.Join(ctx.Spool, nodeId.String(), string(TRx)) - logMsg := func(les LEs) string { - return fmt.Sprintf("Ensuring directory %s existence", dirPath) +func ensureDir(dirs ...string) error { + p := filepath.Join(dirs...) + fi, err := os.Stat(p) + if err == nil { + if fi.IsDir() { + return nil + } + return fmt.Errorf("%s: is not a directory", p) } - if err := os.MkdirAll(dirPath, os.FileMode(0777)); err != nil { - ctx.LogE("dir-ensure-mkdir", LEs{{"Dir", dirPath}}, err, logMsg) + if !os.IsNotExist(err) { return err } - fd, err := os.Open(dirPath) + return os.MkdirAll(p, os.FileMode(0777)) +} + +func (ctx *Ctx) ensureRxDir(nodeId *NodeId) error { + dirPath := filepath.Join(ctx.Spool, nodeId.String(), string(TRx)) + err := ensureDir(dirPath) if err != nil { - ctx.LogE("dir-ensure-open", LEs{{"Dir", dirPath}}, err, logMsg) - return err + ctx.LogE("dir-ensure-mkdir", LEs{{"Dir", dirPath}}, err, func(les LEs) string { + return fmt.Sprintf("Ensuring directory %s existence", dirPath) + }) } - return fd.Close() + return err } func CtxFromCmdline( diff --git a/src/df.go b/src/df.go index 61e2a5c7efd1b8853754bd2f5e33d38bc86d9e952f6a80ea3020fd7bf87a5526..97258a405d938e9ae8a5cc17fbc58319405165d059490afdfa102d728c062cdd 100644 --- a/src/df.go +++ b/src/df.go @@ -1,3 +1,4 @@ +//go:build !netbsd // +build !netbsd // NNCP -- Node to Node copy, utilities for store-and-forward data exchange diff --git a/src/df_netbsd.go b/src/df_netbsd.go index 7bf4fa2dc2da96bea6c7e53eb292d3734c1194a491b70303d80bf37fe1b870c3..16c9374fef2207c9d5ba1cb2e6821c5af8310e615281c85f7a4770344f502a4d 100644 --- a/src/df_netbsd.go +++ b/src/df_netbsd.go @@ -1,3 +1,4 @@ +//go:build netbsd // +build netbsd // NNCP -- Node to Node copy, utilities for store-and-forward data exchange diff --git a/src/dirwatch.go b/src/dirwatch.go new file mode 100644 index 0000000000000000000000000000000000000000..e8624f40f96ee1384649875c2d4aed1d088f716f8a041363be6741658c4de8f0 --- /dev/null +++ b/src/dirwatch.go @@ -0,0 +1,92 @@ +//go:build !nofsnotify +// +build !nofsnotify + +/* +NNCP -- Node to Node copy, utilities for store-and-forward data exchange +Copyright (C) 2016-2021 Sergey Matveev + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, version 3 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package nncp + +import ( + "fmt" + "time" + + "github.com/fsnotify/fsnotify" +) + +type DirWatcher struct { + w *fsnotify.Watcher + C chan struct{} + isDead chan struct{} +} + +func (ctx *Ctx) NewDirWatcher(dir string, d time.Duration) (*DirWatcher, error) { + w, err := fsnotify.NewWatcher() + if err != nil { + return nil, err + } + err = ensureDir(dir) + if err != nil { + return nil, err + } + err = w.Add(dir) + if err != nil { + w.Close() + return nil, err + } + dw := DirWatcher{ + w: w, + C: make(chan struct{}), + isDead: make(chan struct{}), + } + go func() { + ticker := time.NewTicker(d) + dw.C <- struct{}{} + hasEvents := false + for { + select { + case err := <-w.Errors: + ctx.LogE("dir-watch", LEs{{"Dir", dir}}, err, func(les LEs) string { + return "fsnotify error: " + err.Error() + }) + case e := <-w.Events: + ctx.LogD("dir-watch-event", LEs{{"Dir", dir}}, func(les LEs) string { + return fmt.Sprintf("fsnotify event: %v", e) + }) + if e.Op&(fsnotify.Create|fsnotify.Rename) > 0 { + hasEvents = true + } + case <-ticker.C: + if hasEvents { + dw.C <- struct{}{} + hasEvents = false + } + case <-dw.isDead: + w.Close() + ticker.Stop() + close(dw.C) + return + } + } + }() + return &dw, err +} + +func (dw *DirWatcher) Close() { + close(dw.isDead) + for range dw.C { + } +} diff --git a/src/dirwatch_dummy.go b/src/dirwatch_dummy.go new file mode 100644 index 0000000000000000000000000000000000000000..d1b80f1583332761221500979b2ad596690aaa51a917c554385536372b526c3c --- /dev/null +++ b/src/dirwatch_dummy.go @@ -0,0 +1,46 @@ +//go:build nofsnotify +// +build nofsnotify + +/* +NNCP -- Node to Node copy, utilities for store-and-forward data exchange +Copyright (C) 2016-2021 Sergey Matveev + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, version 3 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package nncp + +import ( + "time" +) + +type DirWatcher struct { + C chan struct{} + ticker *time.Ticker +} + +func (ctx *Ctx) NewDirWatcher(dir string, d time.Duration) (*DirWatcher, error) { + dw := DirWatcher{C: make(chan struct{}), ticker: time.NewTicker(d)} + go func() { + for range dw.ticker.C { + dw.C <- struct{}{} + } + }() + return &dw, nil +} + +func (dw *DirWatcher) Close() { + dw.ticker.Stop() + for range dw.C { + } +} diff --git a/src/go.mod b/src/go.mod index 95fdd47dcfc84c2e01587f9fcc301090ca7e591acc181ad04121f2d04cc72610..90fb9e81b99a57df6987a5a2a375b62ed210ca55bf06bde049eb108bc196101f 100644 --- a/src/go.mod +++ b/src/go.mod @@ -4,6 +4,7 @@ require ( github.com/davecgh/go-xdr v0.0.0-20161123171359-e6a2ba005892 github.com/dustin/go-humanize v1.0.0 github.com/flynn/noise v1.0.0 + github.com/fsnotify/fsnotify v1.5.1 github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75 github.com/hjson/hjson-go v3.1.0+incompatible github.com/klauspost/compress v1.13.1 @@ -11,7 +12,7 @@ go.cypherpunks.ru/balloon v1.1.1 go.cypherpunks.ru/recfile v0.4.3 golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 golang.org/x/net v0.0.0-20210614182718-04defd469f4e - golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c + golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b lukechampine.com/blake3 v1.1.5 ) diff --git a/src/go.sum b/src/go.sum index a01fb33ba3a770dd3c9b67f5f923b6c32b1422d096e756e5fe8dca9589dd7f3a..784be2e032486a4daf89f1cf0ade26663ac56d1d5d82143703aabfbcebc233ba 100644 --- a/src/go.sum +++ b/src/go.sum @@ -4,6 +4,8 @@ github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/flynn/noise v1.0.0 h1:DlTHqmzmvcEiKj+4RYo/imoswx/4r6iBlCMfVtrMXpQ= github.com/flynn/noise v1.0.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag= +github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= +github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75 h1:f0n1xnMSmBLzVfsMMvriDyA75NB/oBgILX2GcHXIQzY= @@ -32,8 +34,9 @@ golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf h1:2ucpDCmfkl8Bd/FsLtiD653Wf96cW37s+iGx93zsu4k= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= diff --git a/src/jobs.go b/src/jobs.go index a1197f64c8a2c7745afb9f8e00dee2e784911154ba9107232c48db4faec44eb1..255b245147b65f38553e2ec8007ea78475a1a76abde17479f6a0f6e63fe15c7c 100644 --- a/src/jobs.go +++ b/src/jobs.go @@ -35,7 +35,7 @@ const ( TRx TRxTx = "rx" TTx TRxTx = "tx" - HdrSuffix = ".hdr" + HdrDir = "hdr" ) type Job struct { @@ -43,6 +43,10 @@ PktEnc *PktEnc Path string Size int64 HshValue *[MTHSize]byte +} + +func JobPath2Hdr(jobPath string) string { + return filepath.Join(filepath.Dir(jobPath), HdrDir, filepath.Base(jobPath)) } func (ctx *Ctx) HdrRead(r io.Reader) (*PktEnc, []byte, error) { @@ -80,7 +84,13 @@ }) os.Remove(tmpHdr.Name()) return err } - if err = os.Rename(tmpHdr.Name(), tgt+HdrSuffix); err != nil { + if err = ensureDir(filepath.Dir(tgt), HdrDir); err != nil { + ctx.LogE("hdr-write-ensure-mkdir", nil, err, func(les LEs) string { + return "Header writing: ensuring directory" + }) + return err + } + if err = os.Rename(tmpHdr.Name(), JobPath2Hdr(tgt)); err != nil { ctx.LogE("hdr-write-rename", nil, err, func(les LEs) string { return "Header writing: renaming" }) @@ -137,7 +147,7 @@ var fd *os.File if nock || part { fd, err = os.Open(pth) } else { - fd, err = os.Open(pth + HdrSuffix) + fd, err = os.Open(JobPath2Hdr(pth)) if err != nil && os.IsNotExist(err) { hdrExists = false fd, err = os.Open(pth) diff --git a/src/log.go b/src/log.go index 72452bf5c40ea9d8069cb1d3ff6e6e6f8331ae8bea6305b02ca2555b3234612f..6ef61c92983f74601474c90640cb8d919e465b6c0ada28e36b6b897b7fce0774 100644 --- a/src/log.go +++ b/src/log.go @@ -21,6 +21,7 @@ import ( "bytes" "fmt" "os" + "sync" "time" "go.cypherpunks.ru/recfile" @@ -29,7 +30,10 @@ ) const LogFdPrefix = "FD:" -var LogFd *os.File +var ( + LogFd *os.File + LogFdLock sync.Mutex +) type LE struct { K string @@ -69,7 +73,9 @@ } func (ctx *Ctx) Log(rec string) { if LogFd != nil { + LogFdLock.Lock() LogFd.WriteString(rec) + LogFdLock.Unlock() return } fdLock, err := os.OpenFile( diff --git a/src/nncp.go b/src/nncp.go index f3cf518c8636dd77c4a1d80c745715cb9af61249b3e15b178c750e1c73eeee25..86b75afeaeaaa010ae557559386dd56db0feab25ef48fefff239064befd1e604 100644 --- a/src/nncp.go +++ b/src/nncp.go @@ -40,7 +40,7 @@ const Base32Encoded32Len = 52 var ( - Version string = "7.6.0" + Version string = "7.7.0" Base32Codec *base32.Encoding = base32.StdEncoding.WithPadding(base32.NoPadding) ) diff --git a/src/sp.go b/src/sp.go index 8c77031da21b9edea428094224b48774dac5dbf10251210b4e564d30b3d190ea..a7b699e3d8f791ba6fa5a8a78d0e89aac6985556c74dc21d1e09f52407f867bd 100644 --- a/src/sp.go +++ b/src/sp.go @@ -23,6 +23,7 @@ "crypto/subtle" "errors" "fmt" "io" + "log" "os" "path/filepath" "sort" @@ -772,14 +773,21 @@ // Spool checker and INFOs sender of appearing files if !state.listOnly && (state.xxOnly == "" || state.xxOnly == TTx) { state.wg.Add(1) go func() { - ticker := time.NewTicker(time.Second) + dw, err := state.Ctx.NewDirWatcher( + filepath.Join(state.Ctx.Spool, state.Node.Id.String(), string(TTx)), + time.Second, + ) + if err != nil { + state.Ctx.LogE("sp-queue-dir-watch", les, err, logMsg) + log.Fatalln(err) + } for { select { case <-state.isDead: + dw.Close() state.wg.Done() - ticker.Stop() return - case <-ticker.C: + case <-dw.C: for _, payload := range state.Ctx.infosOur( state.Node.Id, state.Nice, @@ -1255,7 +1263,10 @@ replies = append(replies, MarshalSP(SPTypeDone, SPDone{info.Hash})) } continue } - if _, err = os.Stat(pktPath + SeenSuffix); err == nil { + if _, err = os.Stat(filepath.Join( + state.Ctx.Spool, state.Node.Id.String(), string(TRx), + SeenDir, Base32Codec.EncodeToString(info.Hash[:]), + )); err == nil { state.Ctx.LogI("sp-info-seen", lesp, func(les LEs) string { return logMsg(les) + ": already seen" }) @@ -1576,7 +1587,7 @@ state.Ctx.LogI("sp-done", lesp, func(les LEs) string { return fmt.Sprintf("Packet %s is sent", pktName) }) if state.Ctx.HdrUsage { - os.Remove(pth + HdrSuffix) + os.Remove(JobPath2Hdr(pth)) } } else { state.Ctx.LogE("sp-done", lesp, err, logMsg) diff --git a/src/tmp.go b/src/tmp.go index dac13781403a1e81d61eb5ddd32533387f9d878f1a0b6c6d6e6a65305b97d232..cda429463e02b503043cabb2ba1f3ab316221ee99e6e6a41659b8d61120b53fb 100644 --- a/src/tmp.go +++ b/src/tmp.go @@ -37,7 +37,7 @@ } func (ctx *Ctx) NewTmpFile() (*os.File, error) { jobsPath := filepath.Join(ctx.Spool, "tmp") - if err := os.MkdirAll(jobsPath, os.FileMode(0777)); err != nil { + if err := ensureDir(jobsPath); err != nil { return nil, err } fd, err := TempFile(jobsPath, "") @@ -95,7 +95,7 @@ } func (tmp *TmpFileWHash) Commit(dir string) error { var err error - if err = os.MkdirAll(dir, os.FileMode(0777)); err != nil { + if err = ensureDir(dir); err != nil { return err } if err = tmp.W.Flush(); err != nil { diff --git a/src/toss.go b/src/toss.go index bbc14cd2193b9f4eeccef0e98bd456bb3036d73f0b02cadab93affa8d4704298..4be3a5ca7b092d83b65a8931c6da3a0e41789cc8a6bb6aaf1d6e75f9dbd92998 100644 --- a/src/toss.go +++ b/src/toss.go @@ -43,8 +43,12 @@ "golang.org/x/crypto/poly1305" ) const ( - SeenSuffix = ".seen" + SeenDir = "seen" ) + +func jobPath2Seen(jobPath string) string { + return filepath.Join(filepath.Dir(jobPath), SeenDir, filepath.Base(jobPath)) +} func newNotification(fromTo *FromToJSON, subject string, body []byte) io.Reader { lines := []string{ @@ -198,7 +202,10 @@ ) }) if !dryRun && jobPath != "" { if doSeen { - if fd, err := os.Create(jobPath + SeenSuffix); err == nil { + if err := ensureDir(filepath.Dir(jobPath), SeenDir); err != nil { + return err + } + if fd, err := os.Create(jobPath2Seen(jobPath)); err == nil { fd.Close() if err = DirSync(filepath.Dir(jobPath)); err != nil { ctx.LogE("rx-dirsync", les, err, func(les LEs) string { @@ -223,7 +230,7 @@ ) }) return err } else if ctx.HdrUsage { - os.Remove(jobPath + HdrSuffix) + os.Remove(JobPath2Hdr(jobPath)) } } @@ -391,7 +398,10 @@ }) if !dryRun { if jobPath != "" { if doSeen { - if fd, err := os.Create(jobPath + SeenSuffix); err == nil { + if err := ensureDir(filepath.Dir(jobPath), SeenDir); err != nil { + return err + } + if fd, err := os.Create(jobPath2Seen(jobPath)); err == nil { fd.Close() if err = DirSync(filepath.Dir(jobPath)); err != nil { ctx.LogE("rx-dirsync", les, err, func(les LEs) string { @@ -416,7 +426,7 @@ ) }) return err } else if ctx.HdrUsage { - os.Remove(jobPath + HdrSuffix) + os.Remove(JobPath2Hdr(jobPath)) } } if len(sendmail) > 0 && ctx.NotifyFile != nil { @@ -516,7 +526,10 @@ }) if !dryRun { if jobPath != "" { if doSeen { - if fd, err := os.Create(jobPath + SeenSuffix); err == nil { + if err := ensureDir(filepath.Dir(jobPath), SeenDir); err != nil { + return err + } + if fd, err := os.Create(jobPath2Seen(jobPath)); err == nil { fd.Close() if err = DirSync(filepath.Dir(jobPath)); err != nil { ctx.LogE("rx-dirsync", les, err, func(les LEs) string { @@ -541,7 +554,7 @@ ) }) return err } else if ctx.HdrUsage { - os.Remove(jobPath + HdrSuffix) + os.Remove(JobPath2Hdr(jobPath)) } } if len(sendmail) > 0 && ctx.NotifyFreq != nil { @@ -629,7 +642,10 @@ ) }) if !dryRun && jobPath != "" { if doSeen { - if fd, err := os.Create(jobPath + SeenSuffix); err == nil { + if err := ensureDir(filepath.Dir(jobPath), SeenDir); err != nil { + return err + } + if fd, err := os.Create(jobPath2Seen(jobPath)); err == nil { fd.Close() if err = DirSync(filepath.Dir(jobPath)); err != nil { ctx.LogE("rx-dirsync", les, err, func(les LEs) string { @@ -655,7 +671,7 @@ ) }) return err } else if ctx.HdrUsage { - os.Remove(jobPath + HdrSuffix) + os.Remove(JobPath2Hdr(jobPath)) } } @@ -698,7 +714,7 @@ lesEcho := append(les, LE{"Echo", nodeId}) seenDir := filepath.Join( ctx.Spool, nodeId.String(), AreaDir, area.Id.String(), ) - seenPath := filepath.Join(seenDir, msgHash+SeenSuffix) + seenPath := filepath.Join(seenDir, msgHash) logMsgNode := func(les LEs) string { return fmt.Sprintf( "%s: echoing to: %s", logMsg(les), node.Name, @@ -719,7 +735,7 @@ lesEcho := append(les, LE{"Echo", nodeId}) seenDir := filepath.Join( ctx.Spool, nodeId.String(), AreaDir, area.Id.String(), ) - seenPath := filepath.Join(seenDir, msgHash+SeenSuffix) + seenPath := filepath.Join(seenDir, msgHash) logMsgNode := func(les LEs) string { return fmt.Sprintf("%s: echo to: %s", logMsg(les), node.Name) } @@ -759,7 +775,7 @@ seenDir := filepath.Join( ctx.Spool, ctx.SelfId.String(), AreaDir, area.Id.String(), ) - seenPath := filepath.Join(seenDir, msgHash+SeenSuffix) + seenPath := filepath.Join(seenDir, msgHash) if _, err := os.Stat(seenPath); err == nil { ctx.LogD("rx-area-seen", les, func(les LEs) string { return logMsg(les) + ": already seen" @@ -776,7 +792,7 @@ ) }) return err } else if ctx.HdrUsage { - os.Remove(jobPath + HdrSuffix) + os.Remove(JobPath2Hdr(jobPath)) } } return nil @@ -876,7 +892,7 @@ ) }) return err } else if ctx.HdrUsage { - os.Remove(jobPath + HdrSuffix) + os.Remove(JobPath2Hdr(jobPath)) } } @@ -1020,6 +1036,13 @@ nodeId *NodeId, nice uint8, doSeen, noFile, noFreq, noExec, noTrns, noArea bool, ) (chan struct{}, chan bool) { + dw, err := ctx.NewDirWatcher( + filepath.Join(ctx.Spool, nodeId.String(), string(TRx)), + time.Second, + ) + if err != nil { + log.Fatalln(err) + } finish := make(chan struct{}) badCode := make(chan bool) go func() { @@ -1027,14 +1050,14 @@ bad := false for { select { case <-finish: + dw.Close() badCode <- bad - break - default: + return + case <-dw.C: + bad = !ctx.Toss( + nodeId, TRx, nice, false, + doSeen, noFile, noFreq, noExec, noTrns, noArea) || bad } - time.Sleep(time.Second) - bad = !ctx.Toss( - nodeId, TRx, nice, false, - doSeen, noFile, noFreq, noExec, noTrns, noArea) || bad } }() return finish, badCode diff --git a/src/tx.go b/src/tx.go index ebad936a60f9f21cc0d1fb8a9b506ed71b57bbc1a937dfee0074c9aa4ec80075..c5ad3b77fcef934a695e8d56d2310a4e4700d699c94d6eb868611333ef6ebb24 100644 --- a/src/tx.go +++ b/src/tx.go @@ -239,7 +239,7 @@ msgHash := Base32Codec.EncodeToString(msgHashRaw[:]) seenDir := filepath.Join( ctx.Spool, ctx.SelfId.String(), AreaDir, areaId.String(), ) - seenPath := filepath.Join(seenDir, msgHash+SeenSuffix) + seenPath := filepath.Join(seenDir, msgHash) les := LEs{ {"Node", node.Id}, {"Nice", int(nice)}, @@ -257,7 +257,7 @@ area.Name, msgHash, ) } - if err = os.MkdirAll(seenDir, os.FileMode(0777)); err != nil { + if err = ensureDir(seenDir); err != nil { ctx.LogE("tx-mkdir", les, err, logMsg) return lastNode, err }