X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=t%2Fv2reindex.t;h=990513a3ac07d024d4d1742a864aaa81105ef2c9;hb=0f81681256c9f24c37759e26c6c21db187374438;hp=2a6fc555793d686119892c1a0082d1f07e39fbdd;hpb=0b1de991a099b5e8b9a9e3e85b5eaaacc9362dbb;p=public-inbox.git diff --git a/t/v2reindex.t b/t/v2reindex.t index 2a6fc555..990513a3 100644 --- a/t/v2reindex.t +++ b/t/v2reindex.t @@ -1,23 +1,18 @@ -# Copyright (C) 2018 all contributors +# Copyright (C) 2018-2019 all contributors # License: AGPL-3.0+ use strict; use warnings; use Test::More; use PublicInbox::MIME; use PublicInbox::ContentId qw(content_digest); -use File::Temp qw/tempdir/; use File::Path qw(remove_tree); -require './t/common.perl'; +use PublicInbox::TestCommon; require_git(2.6); - -foreach my $mod (qw(DBD::SQLite Search::Xapian)) { - eval "require $mod"; - plan skip_all => "$mod missing for v2reindex.t" if $@; -} +require_mods(qw(DBD::SQLite Search::Xapian)); use_ok 'PublicInbox::V2Writable'; -my $mainrepo = tempdir('pi-v2reindex-XXXXXX', TMPDIR => 1, CLEANUP => 1); +my ($inboxdir, $for_destroy) = tmpdir(); my $ibx_config = { - mainrepo => $mainrepo, + inboxdir => $inboxdir, name => 'test-v2writable', version => 2, -primary_address => 'test@example.com', @@ -100,7 +95,7 @@ my ($mark1, $mark2, $mark3, $mark4); is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged'); } -my $xap = "$mainrepo/xap".PublicInbox::Search::SCHEMA_VERSION(); +my $xap = "$inboxdir/xap".PublicInbox::Search::SCHEMA_VERSION(); remove_tree($xap); ok(!-d $xap, 'Xapian directories removed'); { @@ -120,7 +115,7 @@ ok(!-d $xap, 'Xapian directories removed'); is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged'); } -ok(unlink "$mainrepo/msgmap.sqlite3", 'remove msgmap'); +ok(unlink "$inboxdir/msgmap.sqlite3", 'remove msgmap'); remove_tree($xap); ok(!-d $xap, 'Xapian directories removed again'); { @@ -143,7 +138,7 @@ ok(!-d $xap, 'Xapian directories removed again'); } my %sizes; -ok(unlink "$mainrepo/msgmap.sqlite3", 'remove msgmap'); +ok(unlink "$inboxdir/msgmap.sqlite3", 'remove msgmap'); remove_tree($xap); ok(!-d $xap, 'Xapian directories removed again'); { @@ -162,13 +157,13 @@ ok(!-d $xap, 'Xapian directories removed again'); is($ibx->mm->num_highwater, 10, 'num_highwater as expected'); my $mset = $ibx->search->query($phrase, {mset=>1}); isnt($mset->size, 0, "phrase search succeeds on indexlevel=full"); - for (<"$xap/*/*">) { $sizes{$ibx->{indexlevel}} += -s _ if -f $_ } + for (glob("$xap/*/*")) { $sizes{$ibx->{indexlevel}} += -s _ if -f $_ } my ($min, $max) = $ibx->mm->minmax; is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged'); } -ok(unlink "$mainrepo/msgmap.sqlite3", 'remove msgmap'); +ok(unlink "$inboxdir/msgmap.sqlite3", 'remove msgmap'); remove_tree($xap); ok(!-d $xap, 'Xapian directories removed again'); { @@ -198,7 +193,7 @@ ok(!-d $xap, 'Xapian directories removed again'); $words =~ tr/"'//d; my $mset = $ibx->search->query($words, {mset=>1}); isnt($mset->size, 0, "normal search works on indexlevel=medium"); - for (<"$xap/*/*">) { $sizes{$ibx->{indexlevel}} += -s _ if -f $_ } + for (glob("$xap/*/*")) { $sizes{$ibx->{indexlevel}} += -s _ if -f $_ } ok($sizes{full} > $sizes{medium}, 'medium is smaller than full'); @@ -207,7 +202,7 @@ ok(!-d $xap, 'Xapian directories removed again'); is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged'); } -ok(unlink "$mainrepo/msgmap.sqlite3", 'remove msgmap'); +ok(unlink "$inboxdir/msgmap.sqlite3", 'remove msgmap'); remove_tree($xap); ok(!-d $xap, 'Xapian directories removed again'); { @@ -228,7 +223,7 @@ ok(!-d $xap, 'Xapian directories removed again'); isnt($ibx->search, 'no search for basic'); - for (<"$xap/*/*">) { $sizes{$ibx->{indexlevel}} += -s _ if -f $_ } + for (glob("$xap/*/*")) { $sizes{$ibx->{indexlevel}} += -s _ if -f $_ } ok($sizes{medium} > $sizes{basic}, 'basic is smaller than medium'); my ($min, $max) = $ibx->mm->minmax; @@ -237,7 +232,7 @@ ok(!-d $xap, 'Xapian directories removed again'); # An incremental indexing test -ok(unlink "$mainrepo/msgmap.sqlite3", 'remove msgmap'); +ok(unlink "$inboxdir/msgmap.sqlite3", 'remove msgmap'); remove_tree($xap); ok(!-d $xap, 'Xapian directories removed again'); { @@ -347,7 +342,7 @@ ok(!-d $xap, 'Xapian directories removed again'); # Another incremental indexing test -ok(unlink "$mainrepo/msgmap.sqlite3", 'remove msgmap'); +ok(unlink "$inboxdir/msgmap.sqlite3", 'remove msgmap'); remove_tree($xap); ok(!-d $xap, 'Xapian directories removed again'); { @@ -431,4 +426,78 @@ ok(!-d $xap, 'Xapian directories removed again'); ], 'msgmap as expected' ); } +# A real example from linux-renesas-soc on lore where a 3-headed monster +# of a message has 3 sets of common headers. Another normal message +# previously existed with a single Message-ID that conflicts with one +# of the Message-IDs in the 3-headed monster. +{ + my @warn; + local $SIG{__WARN__} = sub { push @warn, @_ }; + my %config = %$ibx_config; + $config{indexlevel} = 'medium'; + my $ibx = PublicInbox::Inbox->new(\%config); + my $im = PublicInbox::V2Writable->new($ibx); + my $m3 = PublicInbox::MIME->new(<<'EOF'); +Date: Tue, 24 May 2016 14:34:22 -0700 (PDT) +Message-Id: <20160524.143422.552507610109476444.d@example.com> +To: t@example.com +Cc: c@example.com +Subject: Re: [PATCH v2 2/2] uno +From: +In-Reply-To: <1463825855-7363-2-git-send-email-y@example.com> +References: <1463825855-7363-1-git-send-email-y@example.com> + <1463825855-7363-2-git-send-email-y@example.com> +Date: Wed, 25 May 2016 10:01:51 +0900 +From: h@example.com +To: g@example.com +Cc: m@example.com +Subject: Re: [PATCH] dos +Message-ID: <20160525010150.GD7292@example.com> +References: <1463498133-23918-1-git-send-email-g+r@example.com> +In-Reply-To: <1463498133-23918-1-git-send-email-g+r@example.com> +From: s@example.com +To: h@example.com +Cc: m@example.com +Subject: [PATCH 12/13] tres +Date: Wed, 01 Jun 2016 01:32:35 +0300 +Message-ID: <1923946.Jvi0TDUXFC@wasted.example.com> +In-Reply-To: <13205049.n7pM8utpHF@wasted.example.com> +References: <13205049.n7pM8utpHF@wasted.example.com> + +Somehow we got a message with 3 sets of headers into one +message, could've been something broken on the archiver side. +EOF + + my $m1 = PublicInbox::MIME->new(<<'EOF'); +From: a@example.com +To: t@example.com +Subject: [PATCH 12/13] +Date: Wed, 01 Jun 2016 01:32:35 +0300 +Message-ID: <1923946.Jvi0TDUXFC@wasted.example.com> +In-Reply-To: <13205049.n7pM8utpHF@wasted.example.com> +References: <13205049.n7pM8utpHF@wasted.example.com> + +This is probably one of the original messages + +EOF + $im->add($m1); + $im->add($m3); + $im->done; + remove_tree($xap); + eval { $im->index_sync() }; + is($@, '', 'no error from initial indexing'); + is_deeply(\@warn, [], 'no warnings from initial index'); + eval { $im->index_sync({reindex=>1}) }; + is($@, '', 'no error from reindexing after reused Message-ID (x3)'); + is_deeply(\@warn, [], 'no warnings on reindex'); + + my %uniq; + for my $s (qw(uno dos tres)) { + my $msgs = $ibx->search->query("s:$s"); + is(scalar(@$msgs), 1, "only one result for `$s'"); + $uniq{$msgs->[0]->{num}}++; + } + is_deeply([values %uniq], [3], 'search on different subjects'); +} + done_testing();