lei <q|up>: distinguish between mset and l2m counts

The number of messages we write to --output is usually different
than the mset count due to deduplication from combining multiple
sources.

This change makes the stderr output of "lei up --all=local" way
more useful IMHO.

diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index eda4701c595d28962660d0d39719e68001523886..a546ab421eeb7c72282f1e72de3a811fbad1f2ed 100644 (file)
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -196,9 +196,13 @@ sub _mbox_write_cb ($$) {
                return if $dedupe->is_dup($eml, $smsg);
                $lse->xsmsg_vmd($smsg) if $lse;
                $buf = $eml2mbox->($eml, $smsg);
-               return atomic_append($lei, $buf) if $atomic_append;
-               my $lk = $ovv->lock_for_scope;
-               $lei->out($$buf);
+               if ($atomic_append) {
+                       atomic_append($lei, $buf);
+               } else {
+                       my $lk = $ovv->lock_for_scope;
+                       $lei->out($$buf);
+               }
+               ++$lei->{-nr_write};
        }
 }
 
@@ -273,15 +277,13 @@ sub _maildir_write_cb ($$) {
        my $dst = $lei->{ovv}->{dst};
        my $lse = $lei->{lse}; # may be undef
        sub { # for git_to_mail
-               my ($buf, $smsg, $eml) = @_;
+               my ($bref, $smsg, $eml) = @_;
                $dst // return $lei->fail; # dst may be undef-ed in last run
-               $buf //= \($eml->as_string);
+               return if $dedupe && $dedupe->is_dup($eml //
+                                               PublicInbox::Eml->new($$bref));
                $lse->xsmsg_vmd($smsg) if $lse;
-               return _buf2maildir($dst, $buf, $smsg) if !$dedupe;
-               $eml //= PublicInbox::Eml->new($$buf); # copy buf
-               return if $dedupe->is_dup($eml, $smsg);
-               undef $eml;
-               _buf2maildir($dst, $buf, $smsg);
+               _buf2maildir($dst, $bref // \($eml->as_string), $smsg);
+               ++$lei->{-nr_write};
        }
 }
 
@@ -296,16 +298,15 @@ sub _imap_write_cb ($$) {
        sub { # for git_to_mail
                my ($bref, $smsg, $eml) = @_;
                $mic // return $lei->fail; # mic may be undef-ed in last run
-               if ($dedupe) {
-                       $eml //= PublicInbox::Eml->new($$bref); # copy bref
-                       return if $dedupe->is_dup($eml, $smsg);
-               }
+               return if $dedupe && $dedupe->is_dup($eml //
+                                               PublicInbox::Eml->new($$bref));
                $lse->xsmsg_vmd($smsg) if $lse;
                eval { $imap_append->($mic, $folder, $bref, $smsg, $eml) };
                if (my $err = $@) {
                        undef $mic;
                        die $err;
                }
+               ++$lei->{-nr_write};
        }
 }
 
@@ -659,7 +660,12 @@ sub write_mail { # via ->wq_io_do
 sub wq_atexit_child {
        my ($self) = @_;
        delete $self->{wcb};
-       $self->{lei}->{ale}->git->async_wait_all;
+       my $lei = $self->{lei};
+       $lei->{ale}->git->async_wait_all;
+       my $nr = delete($lei->{-nr_write}) or return;
+       return if $lei->{early_mua} || !$lei->{-progress} || !$lei->{pkt_op_p};
+       require PublicInbox::PktOp;
+       PublicInbox::PktOp::pkt_do($lei->{pkt_op_p}, 'l2m_progress', $nr);
 }
 
 # called in top-level lei-daemon when LeiAuth is done

diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index b3fd79d005c9ad00954c6d6e9b5edfce4e653c3d..e04af0dc8a05d00034a42b3e9f71ba488fbdc0a3 100644 (file)
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -144,6 +144,11 @@ sub mset_progress {
        }
 }
 
+sub l2m_progress {
+       my ($lei, $nr) = @_;
+       $lei->{-nr_write} += $nr;
+}
+
 sub query_one_mset { # for --threads and l2m w/o sort
        my ($self, $ibxish) = @_;
        local $0 = "$0 query_one_mset";
@@ -354,7 +359,7 @@ sub query_done { # EOF callback for main daemon
        }
        my $wait = $lei->{sto} ? $lei->{sto}->ipc_do('done') : undef;
        $lei->{ovv}->ovv_end($lei);
-       my @out;
+       my (@out, $start_mua);
        if ($l2m) { # close() calls LeiToMail reap_compress
                @out = (" in $lei->{ovv}->{dst}");
                if (my $out = delete $lei->{old_1}) {
@@ -370,11 +375,15 @@ Error closing $lei->{ovv}->{dst}: $!
                        $lei->poke_mua;
                } else { # mbox users
                        delete $l2m->{mbl}; # drop dotlock
-                       $lei->start_mua;
+                       $start_mua = 1;
                }
        }
-       $lei->{-progress} and
-               $lei->err('# ', $lei->{-mset_total} // 0, " matches", @out);
+       if ($lei->{-progress}) {
+               $lei->qerr('# ', $lei->{-mset_total} // 0, " matches", @out);
+               my $nr = $lei->{-nr_write} // 0;
+               $lei->qerr("# $nr written to $lei->{ovv}->{dst}") if $l2m;
+       }
+       $lei->start_mua if $start_mua;
        $lei->dclose;
 }
 
@@ -456,6 +465,7 @@ sub do_query {
                '+' => [ \&incr_post_augment, $lei ],
                '' => [ \&query_done, $lei ],
                'mset_progress' => [ \&mset_progress, $lei ],
+               'l2m_progress' => [ \&l2m_progress, $lei ],
                'x_it' => [ $lei->can('x_it'), $lei ],
                'child_error' => [ $lei->can('child_error'), $lei ],
                'incr_start_query' => [ \&incr_start_query, $self, $l2m ],