truncate Message-IDs and References consistently

We need to stop ghost messages from generating longer
Message-IDs than Xapian can handle with terms.

diff --git a/lib/PublicInbox/MID.pm b/lib/PublicInbox/MID.pm
index 117d3c42f987e1fc2079cbccd2025f0176194085..c82e84013ec0d68730423ab4b691074c5b690dc5 100644 (file)
--- a/lib/PublicInbox/MID.pm
+++ b/lib/PublicInbox/MID.pm
@@ -65,12 +65,6 @@ sub mids ($) {
                        push(@mids, $v);
                }
        }
-       foreach my $i (0..$#mids) {
-               next if length($mids[$i]) <= MAX_MID_SIZE;
-               warn "Message-ID: <$mids[$i]> too long, truncating\n";
-               $mids[$i] = substr($mids[$i], 0, MAX_MID_SIZE);
-       }
-
        uniq_mids(\@mids);
 }
 
@@ -92,10 +86,14 @@ sub uniq_mids ($) {
        my ($mids) = @_;
        my @ret;
        my %seen;
-       foreach (@$mids) {
-               next if $seen{$_};
-               push @ret, $_;
-               $seen{$_} = 1;
+       foreach my $mid (@$mids) {
+               if (length($mid) > MAX_MID_SIZE) {
+                       warn "Message-ID: <$mid> too long, truncating\n";
+                       $mid = substr($mid, 0, MAX_MID_SIZE);
+               }
+               next if $seen{$mid};
+               push @ret, $mid;
+               $seen{$mid} = 1;
        }
        \@ret;
 }

diff --git a/t/v2writable.t b/t/v2writable.t
index 4a7cfb909ee6f3030d722a532a48e7174fad44d9..7e29ef7660de98376346f1b25c1615351098733e 100644 (file)
--- a/t/v2writable.t
+++ b/t/v2writable.t
@@ -235,4 +235,26 @@ EOF
        $im->done;
 }
 
+{
+       my @warn;
+       my $x = 'x'x250;
+       my $y = 'y'x250;
+       local $SIG{__WARN__} = sub { push @warn, @_ };
+       $mime->header_set('Subject', 'long mid');
+       $mime->header_set('Message-ID', "<$x>");
+       ok($im->add($mime), 'add excessively long Message-ID');
+
+       $mime->header_set('Message-ID', "<$y>");
+       $mime->header_set('References', "<$x>");
+       ok($im->add($mime), 'add excessively long References');
+       $im->barrier;
+
+       my $msgs = $ibx->search->reopen->get_thread('x'x244)->{msgs};
+       is(2, scalar(@$msgs), 'got both messages');
+       is($msgs->[0]->{mid}, 'x'x244, 'stored truncated mid');
+       is($msgs->[1]->{references}, '<'.('x'x244).'>', 'stored truncated ref');
+       is($msgs->[1]->{mid}, 'y'x244, 'stored truncated mid(2)');
+       $im->done;
+}
+
 done_testing();