index: allow search/lookups on X-Alt-Message-ID

Since we replace extra Message-ID headers with X-Alt-Message-ID
to placate NNTP clients, we should allow searching and indexing
on X-Alt-Message-ID just like we do with Message-ID.

diff --git a/lib/PublicInbox/MID.pm b/lib/PublicInbox/MID.pm
index 14089f914ab8923c129fd1249ddfc87c75989ac3..d7a42c38573b17111458ddccd32e821355f79b91 100644 (file)
--- a/lib/PublicInbox/MID.pm
+++ b/lib/PublicInbox/MID.pm
@@ -7,7 +7,7 @@ use strict;
 use warnings;
 use base qw/Exporter/;
 our @EXPORT_OK = qw/mid_clean id_compress mid2path mid_mime mid_escape MID_ESC
-       mids references/;
+       mids references mids_for_index/;
 use URI::Escape qw(uri_escape_utf8);
 use Digest::SHA qw/sha1_hex/;
 require PublicInbox::Address;
@@ -54,11 +54,10 @@ sub mid2path {
 # Only for v1 code paths:
 sub mid_mime ($) { mids($_[0]->header_obj)->[0] }
 
-sub mids ($) {
-       my ($hdr) = @_;
+# only intended for Message-ID and X-Alt-Message-ID
+sub extract_mids {
        my @mids;
-       my @v = $hdr->header_raw('Message-Id');
-       foreach my $v (@v) {
+       for my $v (@_) {
                my @cur = ($v =~ /<([^>]+)>/sg);
                if (@cur) {
                        push(@mids, @cur);
@@ -66,7 +65,23 @@ sub mids ($) {
                        push(@mids, $v);
                }
        }
-       uniq_mids(\@mids);
+       \@mids;
+}
+
+sub mids ($) {
+       my ($hdr) = @_;
+       my @mids = $hdr->header_raw('Message-Id');
+       uniq_mids(extract_mids(@mids));
+}
+
+# we allow searching on X-Alt-Message-ID since PublicInbox::NNTP uses them
+# to placate some clients, and we want to ensure NNTP-only clients can
+# import and index without relying on HTTP endpoints
+sub mids_for_index ($) {
+       my ($hdr) = @_;
+       my @mids = $hdr->header_raw('Message-Id');
+       my @alts = $hdr->header_raw('X-Alt-Message-ID');
+       uniq_mids(extract_mids(@mids, @alts));
 }
 
 # last References should be IRT, but some mail clients do things

diff --git a/lib/PublicInbox/OverIdx.pm b/lib/PublicInbox/OverIdx.pm
index 01ca6f1160ffbfb6092b7941d9a4835cf9df26f7..189bd21d93bc6731192dd26a333cc79f04b89c3e 100644 (file)
--- a/lib/PublicInbox/OverIdx.pm
+++ b/lib/PublicInbox/OverIdx.pm
@@ -13,7 +13,7 @@ use warnings;
 use base qw(PublicInbox::Over);
 use IO::Handle;
 use DBI qw(:sql_types); # SQL_BLOB
-use PublicInbox::MID qw/id_compress mids references/;
+use PublicInbox::MID qw/id_compress mids_for_index references/;
 use PublicInbox::SearchMsg qw(subject_normalized);
 use Compress::Zlib qw(compress);
 use PublicInbox::Search;
@@ -256,7 +256,7 @@ sub add_overview {
                lines => $lines,
                blob => $oid,
        }, 'PublicInbox::SearchMsg';
-       my $mids = mids($mime->header_obj);
+       my $mids = mids_for_index($mime->header_obj);
        my $refs = parse_references($smsg, $mid0, $mids);
        my $subj = $smsg->subject;
        my $xpath;

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index aed3875a037254b94222faa2147e9f17abe31612..b2d71a1f88b462493d3db527c609d00f4e16b31b 100644 (file)
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -12,7 +12,7 @@ use warnings;
 use base qw(PublicInbox::Search PublicInbox::Lock);
 use PublicInbox::MIME;
 use PublicInbox::InboxWritable;
-use PublicInbox::MID qw/mid_clean id_compress mid_mime mids/;
+use PublicInbox::MID qw/mid_clean id_compress mid_mime mids_for_index/;
 use PublicInbox::MsgIter;
 use Carp qw(croak);
 use POSIX qw(strftime);
@@ -344,7 +344,7 @@ sub add_xapian ($$$$$) {
 sub add_message {
        # mime = Email::MIME object
        my ($self, $mime, $bytes, $num, $oid, $mid0) = @_;
-       my $mids = mids($mime->header_obj);
+       my $mids = mids_for_index($mime->header_obj);
        $mid0 = $mids->[0] unless defined $mid0; # v1 compatibility
        unless (defined $num) { # v1
                $self->_msgmap_init;

diff --git a/t/mid.t b/t/mid.t
index 9ad10a9912d17b3bfcfc21d25d3ad49018fe5b5c..98b0c200ed54841290207d99be020a356112c286 100644 (file)
--- a/t/mid.t
+++ b/t/mid.t
@@ -1,7 +1,7 @@
 # Copyright (C) 2016-2019 all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 use Test::More;
-use PublicInbox::MID qw(mid_escape mids references);
+use PublicInbox::MID qw(mid_escape mids references mids_for_index);
 
 is(mid_escape('foo!@(bar)'), 'foo!@(bar)');
 is(mid_escape('foo%!@(bar)'), 'foo%25!@(bar)');
@@ -10,6 +10,7 @@ is(mid_escape('foo%!@(bar)'), 'foo%25!@(bar)');
 {
        use Email::MIME;
        my $mime = Email::MIME->create;
+       $mime->header_set('X-Alt-Message-ID', '<alt-id-for-nntp>');
        $mime->header_set('Message-Id', '<mid-1@a>');
        is_deeply(['mid-1@a'], mids($mime->header_obj), 'mids in common case');
        $mime->header_set('Message-Id', '<mid-1@a>', '<mid-2@b>');
@@ -40,6 +41,10 @@ is(mid_escape('foo%!@(bar)'), 'foo%25!@(bar)');
        $mime->header_set('To', 'u@example.com');
        $mime->header_set('References', '<hello> <world> <n> <u@example.com>');
        is_deeply(references($mime->header_obj), [qw(hello world)]);
+
+       is_deeply([qw(helloworld alt-id-for-nntp)],
+               mids_for_index($mime->header_obj),
+               'X-Alt-Message-ID can be indexed');
 }
 
 done_testing();

diff --git a/t/v2writable.t b/t/v2writable.t
index c2daac2f8484b4feb72ea752948af6cf9ac2cd2f..2b82576878e13b9def584dfe5578352f4c28a4a2 100644 (file)
--- a/t/v2writable.t
+++ b/t/v2writable.t
@@ -115,6 +115,7 @@ if ('ensure git configs are correct') {
 
 {
        $mime->header_set('Message-Id', '<abcde@1>', '<abcde@2>');
+       $mime->header_set('X-Alt-Message-Id', '<alt-id-for-nntp>');
        $mime->header_set('References', '<zz-mid@b>');
        ok($im->add($mime), 'message with multiple Message-ID');
        $im->done;
@@ -127,6 +128,21 @@ if ('ensure git configs are correct') {
        is($mset2->size, 1, 'message found by second MID');
        is((($mset1->items)[0])->get_docid, (($mset2->items)[0])->get_docid,
                'same document') if ($mset1->size);
+
+       my $alt = $srch->reopen->query('m:alt-id-for-nntp', { mset => 1 });
+       is($alt->size, 1, 'message found by alt MID (NNTP)');
+       is((($alt->items)[0])->get_docid, (($mset1->items)[0])->get_docid,
+               'same document') if ($mset1->size);
+       $mime->header_set('X-Alt-Message-Id');
+
+       my %uniq;
+       for my $mid (qw(abcde@1 abcde@2 alt-id-for-nntp)) {
+               my $msgs = $ibx->over->get_thread($mid);
+               my $key = join(' ', sort(map { $_->{num} } @$msgs));
+               $uniq{$key}++;
+       }
+       is(scalar(keys(%uniq)), 1, 'all alt Message-ID queries give same smsg');
+       is_deeply([values(%uniq)], [3], '3 queries, 3 results');
 }
 
 {