From: Eric Wong (Contractor, The Linux Foundation) <e@80x24.org>
Date: Sun, 1 Apr 2018 23:23:44 +0000 (+0000)
Subject: truncate Message-IDs and References consistently
X-Git-Tag: v1.1.0-pre1~80
X-Git-Url: http://www.git.stargrave.org/?p=public-inbox.git;a=commitdiff_plain;h=1cb230c4d47c8ef5d03f0b8de2f8ad93c303d816

truncate Message-IDs and References consistently

We need to stop ghost messages from generating longer
Message-IDs than Xapian can handle with terms.
---

diff --git a/lib/PublicInbox/MID.pm b/lib/PublicInbox/MID.pm
index 117d3c42..c82e8401 100644
--- a/lib/PublicInbox/MID.pm
+++ b/lib/PublicInbox/MID.pm
@@ -65,12 +65,6 @@ sub mids ($) {
 			push(@mids, $v);
 		}
 	}
-	foreach my $i (0..$#mids) {
-		next if length($mids[$i]) <= MAX_MID_SIZE;
-		warn "Message-ID: <$mids[$i]> too long, truncating\n";
-		$mids[$i] = substr($mids[$i], 0, MAX_MID_SIZE);
-	}
-
 	uniq_mids(\@mids);
 }
 
@@ -92,10 +86,14 @@ sub uniq_mids ($) {
 	my ($mids) = @_;
 	my @ret;
 	my %seen;
-	foreach (@$mids) {
-		next if $seen{$_};
-		push @ret, $_;
-		$seen{$_} = 1;
+	foreach my $mid (@$mids) {
+		if (length($mid) > MAX_MID_SIZE) {
+			warn "Message-ID: <$mid> too long, truncating\n";
+			$mid = substr($mid, 0, MAX_MID_SIZE);
+		}
+		next if $seen{$mid};
+		push @ret, $mid;
+		$seen{$mid} = 1;
 	}
 	\@ret;
 }
diff --git a/t/v2writable.t b/t/v2writable.t
index 4a7cfb90..7e29ef76 100644
--- a/t/v2writable.t
+++ b/t/v2writable.t
@@ -235,4 +235,26 @@ EOF
 	$im->done;
 }
 
+{
+	my @warn;
+	my $x = 'x'x250;
+	my $y = 'y'x250;
+	local $SIG{__WARN__} = sub { push @warn, @_ };
+	$mime->header_set('Subject', 'long mid');
+	$mime->header_set('Message-ID', "<$x>");
+	ok($im->add($mime), 'add excessively long Message-ID');
+
+	$mime->header_set('Message-ID', "<$y>");
+	$mime->header_set('References', "<$x>");
+	ok($im->add($mime), 'add excessively long References');
+	$im->barrier;
+
+	my $msgs = $ibx->search->reopen->get_thread('x'x244)->{msgs};
+	is(2, scalar(@$msgs), 'got both messages');
+	is($msgs->[0]->{mid}, 'x'x244, 'stored truncated mid');
+	is($msgs->[1]->{references}, '<'.('x'x244).'>', 'stored truncated ref');
+	is($msgs->[1]->{mid}, 'y'x244, 'stored truncated mid(2)');
+	$im->done;
+}
+
 done_testing();