lei: less error-prone FD mapping

[public-inbox.git] / lib / PublicInbox / LeiOverview.pm

diff --git a/lib/PublicInbox/LeiOverview.pm b/lib/PublicInbox/LeiOverview.pm
index 8799f1ccec56fcd5c06f96bf54507f4608934692..c67e2747edfac4faeb1613abcca10a8263a73775 100644 (file)
--- a/lib/PublicInbox/LeiOverview.pm
+++ b/lib/PublicInbox/LeiOverview.pm
@@ -203,46 +203,56 @@ sub ovv_each_smsg_cb { # runs in wq worker usually
        my ($self, $lei, $ibxish) = @_;
        my $json;
        $lei->{1}->autoflush(1);
+       my $dedupe = $lei->{dedupe} // die 'BUG: {dedupe} missing';
        if (my $pkg = $self->{json}) {
                $json = $pkg->new;
                $json->utf8->canonical;
                $json->ascii(1) if $lei->{opt}->{ascii};
        }
-       my $l2m = $lei->{l2m};
-       if ($l2m && $l2m->{-wq_s1}) {
+       my $l2m = $lei->{l2m} or $dedupe->prepare_dedupe;
+       if ($l2m && !$ibxish) { # remote https?:// mboxrd
+               delete $l2m->{-wq_s1};
+               my $g2m = $l2m->can('git_to_mail');
+               my $wcb = $l2m->write_cb($lei);
+               sub {
+                       my ($smsg, undef, $eml) = @_; # no mitem in $_[1]
+                       $wcb->(undef, $smsg, $eml);
+               };
+       } elsif ($l2m && $l2m->{-wq_s1}) {
                my ($lei_ipc, @io) = $lei->atfork_parent_wq($l2m);
-               # n.b. $io[0] = qry_status_wr, $io[1] = mbox|stdout,
-               # $io[4] becomes a notification pipe that triggers EOF
+               # $io[-1] becomes a notification pipe that triggers EOF
                # in this wq worker when all outstanding ->write_mail
                # calls are complete
-               die "BUG: \$io[4] $io[4] unexpected" if $io[4];
-               pipe($l2m->{each_smsg_done}, $io[4]) or die "pipe: $!";
-               fcntl($io[4], 1031, 4096) if $^O eq 'linux';
+               pipe($l2m->{each_smsg_done}, $io[$#io + 1]) or die "pipe: $!";
+               fcntl($io[-1], 1031, 4096) if $^O eq 'linux'; # F_SETPIPE_SZ
                delete @$lei_ipc{qw(l2m opt mset_opt cmd)};
+               $lei_ipc->{each_smsg_not_done} = $#io;
                my $git = $ibxish->git; # (LeiXSearch|Inbox|ExtSearch)->git
                $self->{git} = $git;
                my $git_dir = $git->{git_dir};
                sub {
                        my ($smsg, $mitem) = @_;
-                       my $kw = []; # TODO get from mitem
-                       $l2m->wq_do('write_mail', \@io, $git_dir,
-                                       $smsg->{blob}, $lei_ipc, $kw)
+                       $smsg->{pct} = get_pct($mitem) if $mitem;
+                       $l2m->wq_do('write_mail', \@io, $git_dir, $smsg,
+                                       $lei_ipc);
                }
        } elsif ($l2m) {
                my $wcb = $l2m->write_cb($lei);
                my $git = $ibxish->git; # (LeiXSearch|Inbox|ExtSearch)->git
                $self->{git} = $git; # for ovv_atexit_child
                my $g2m = $l2m->can('git_to_mail');
+               $dedupe->prepare_dedupe;
                sub {
                        my ($smsg, $mitem) = @_;
-                       my $kw = []; # TODO get from mitem
-                       $git->cat_async($smsg->{blob}, $g2m, [ $wcb, $kw ]);
+                       $smsg->{pct} = get_pct($mitem) if $mitem;
+                       $git->cat_async($smsg->{blob}, $g2m, [ $wcb, $smsg ]);
                };
        } elsif ($self->{fmt} =~ /\A(concat)?json\z/ && $lei->{opt}->{pretty}) {
                my $EOR = ($1//'') eq 'concat' ? "\n}" : "\n},";
                $lei->{ovv_buf} = \(my $buf = '');
                sub { # DIY prettiness :P
                        my ($smsg, $mitem) = @_;
+                       return if $dedupe->is_smsg_dup($smsg);
                        $smsg = _unbless_smsg($smsg, $mitem);
                        $buf .= "{\n";
                        $buf .= join(",\n", map {
@@ -266,7 +276,7 @@ sub ovv_each_smsg_cb { # runs in wq worker usually
                $lei->{ovv_buf} = \(my $buf = '');
                sub {
                        my ($smsg, $mitem) = @_;
-                       delete @$smsg{qw(tid num)};
+                       return if $dedupe->is_smsg_dup($smsg);
                        $buf .= $json->encode(_unbless_smsg(@_)) . $ORS;
                        if (length($buf) > 65536) {
                                my $lk = $self->lock_for_scope;
@@ -274,10 +284,6 @@ sub ovv_each_smsg_cb { # runs in wq worker usually
                                $buf = '';
                        }
                }
-       } elsif ($self->{fmt} eq 'oid') {
-               sub {
-                       my ($smsg, $mitem) = @_;
-               }
        } # else { ...
 }