searchidx: ensure duplicated Message-IDs can be linked together

This allows us to emulate the display of thread-aware MUAs when
multiple messages share the same Message-ID.  This also is a
place where "public-inbox-index --reindex" is useful to fix
existing messages and no schema version bump is necessary.

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 3412a615b66c3bdc2fd2897ad3c5ac6b2010287f..06bce70a4218ba82e3d7e420131e6b58a30c6d16 100644 (file)
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -334,7 +334,7 @@ sub add_message {
                });
 
                # populates smsg->references for smsg->to_doc_data
-               my $refs = parse_references($smsg);
+               my $refs = parse_references($smsg, $mid0, $mids);
                my $data = $smsg->to_doc_data($oid, $mid0);
                foreach my $mid (@$mids) {
                        $tg->index_text($mid, 1, 'XM');
@@ -463,22 +463,23 @@ sub term_generator { # write-only
        $self->{term_generator} = $tg;
 }
 
-sub parse_references ($) {
-       my ($smsg) = @_;
+sub parse_references ($$$) {
+       my ($smsg, $mid0, $mids) = @_;
        my $mime = $smsg->{mime};
        my $hdr = $mime->header_obj;
        my $refs = references($hdr);
+       push(@$refs, @$mids) if scalar(@$mids) > 1;
        return $refs if scalar(@$refs) == 0;
 
-       # prevent circular references via References here:
-       my %mids = map { $_ => 1 } @{mids($hdr)};
+       # prevent circular references here:
+       my %seen = ( $mid0 => 1 );
        my @keep;
        foreach my $ref (@$refs) {
                if (length($ref) > PublicInbox::MID::MAX_MID_SIZE) {
                        warn "References: <$ref> too long, ignoring\n";
                        next;
                }
-               next if $mids{$ref};
+               next if $seen{$ref}++;
                push @keep, $ref;
        }
        $smsg->{references} = '<'.join('> <', @keep).'>' if @keep;

diff --git a/t/psgi_v2.t b/t/psgi_v2.t
index aa3279cc6870743e43f62efb4b8f98141d4a7193..bdf23deb1c0080f382d1634712dc57070f2b2e1c 100644 (file)
--- a/t/psgi_v2.t
+++ b/t/psgi_v2.t
@@ -182,6 +182,9 @@ test_psgi(sub { $www->call(@_) }, sub {
        $res = $cb->(GET('/v2test/reuse@mid/T/'));
        $raw = $res->content;
        like($raw, qr/\b4\+ messages\b/, 'thread overview shown with /T/');
+       my @over = ($raw =~ m/^\d{4}-\d+-\d+\s+\d+:\d+ (.+)$/gm);
+       is_deeply(\@over, [ '<a', '` <a', '` <a', '` <a' ],
+               'duplicate messages share the same root');
 
        $res = $cb->(GET('/v2test/reuse@mid/t/'));
        $raw = $res->content;