lei: reinstate JSON smsg output deduplication

[public-inbox.git] / lib / PublicInbox / LeiOverview.pm

diff --git a/lib/PublicInbox/LeiOverview.pm b/lib/PublicInbox/LeiOverview.pm
index 928d66cbed233d0a83e0e51442a527a86d37e094..880c7acccb09f7d86c5169f9195483184217a6b8 100644 (file)
--- a/lib/PublicInbox/LeiOverview.pm
+++ b/lib/PublicInbox/LeiOverview.pm
@@ -203,12 +203,14 @@ sub ovv_each_smsg_cb { # runs in wq worker usually
        my ($self, $lei, $ibxish) = @_;
        my $json;
        $lei->{1}->autoflush(1);
+       my $dedupe = $lei->{dedupe} // die 'BUG: {dedupe} missing';
        if (my $pkg = $self->{json}) {
                $json = $pkg->new;
                $json->utf8->canonical;
                $json->ascii(1) if $lei->{opt}->{ascii};
+               $lei->{ovv_buf} = \(my $buf = '');
        }
-       my $l2m = $lei->{l2m};
+       my $l2m = $lei->{l2m} or $dedupe->prepare_dedupe;
        if ($l2m && !$ibxish) { # remote https?:// mboxrd
                delete $l2m->{-wq_s1};
                my $g2m = $l2m->can('git_to_mail');
@@ -241,6 +243,7 @@ sub ovv_each_smsg_cb { # runs in wq worker usually
                my $git = $ibxish->git; # (LeiXSearch|Inbox|ExtSearch)->git
                $self->{git} = $git; # for ovv_atexit_child
                my $g2m = $l2m->can('git_to_mail');
+               $dedupe->prepare_dedupe;
                sub {
                        my ($smsg, $mitem) = @_;
                        $smsg->{pct} = get_pct($mitem) if $mitem;
@@ -251,6 +254,7 @@ sub ovv_each_smsg_cb { # runs in wq worker usually
                $lei->{ovv_buf} = \(my $buf = '');
                sub { # DIY prettiness :P
                        my ($smsg, $mitem) = @_;
+                       return if $dedupe->is_smsg_dup($smsg);
                        $smsg = _unbless_smsg($smsg, $mitem);
                        $buf .= "{\n";
                        $buf .= join(",\n", map {
@@ -274,6 +278,7 @@ sub ovv_each_smsg_cb { # runs in wq worker usually
                $lei->{ovv_buf} = \(my $buf = '');
                sub {
                        my ($smsg, $mitem) = @_;
+                       return if $dedupe->is_smsg_dup($smsg);
                        $buf .= $json->encode(_unbless_smsg(@_)) . $ORS;
                        if (length($buf) > 65536) {
                                my $lk = $self->lock_for_scope;