X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=t%2Fv1reindex.t;h=240e28f92d6cd16c486bd731cf98c5cae10ac8be;hb=95bdac7f09c69036efed537a4d03d5bdd2ae4eb6;hp=877e70390910d20fb05ba446eaddffdff78fc7d3;hpb=344684fc77a999ac071f99803a0bff1a96dc5f7c;p=public-inbox.git diff --git a/t/v1reindex.t b/t/v1reindex.t index 877e7039..240e28f9 100644 --- a/t/v1reindex.t +++ b/t/v1reindex.t @@ -1,23 +1,20 @@ -# Copyright (C) 2018 all contributors +# Copyright (C) 2018-2020 all contributors # License: AGPL-3.0+ use strict; use warnings; use Test::More; use PublicInbox::MIME; use PublicInbox::ContentId qw(content_digest); -use File::Temp qw/tempdir/; use File::Path qw(remove_tree); - -foreach my $mod (qw(DBD::SQLite Search::Xapian)) { - eval "require $mod"; - plan skip_all => "$mod missing for v1reindex.t" if $@; -} +use PublicInbox::TestCommon; +require_git(2.6); +require_mods(qw(DBD::SQLite Search::Xapian)); use_ok 'PublicInbox::SearchIdx'; use_ok 'PublicInbox::Import'; -my $mainrepo = tempdir('pi-v1reindex-XXXXXX', TMPDIR => 1, CLEANUP => 1); -is(system(qw(git init -q --bare), $mainrepo), 0); +my ($inboxdir, $for_destroy) = tmpdir(); +is(system(qw(git init -q --bare), $inboxdir), 0); my $ibx_config = { - mainrepo => $mainrepo, + inboxdir => $inboxdir, name => 'test-v1reindex', -primary_address => 'test@example.com', indexlevel => 'full', @@ -33,6 +30,7 @@ my $mime = PublicInbox::MIME->create( ); my $minmax; my $msgmap; +my ($mark1, $mark2, $mark3, $mark4); { my %config = %$ibx_config; my $ibx = PublicInbox::Inbox->new(\%config); @@ -41,13 +39,17 @@ my $msgmap; $mime->header_set('Message-Id', "<$i\@example.com>"); ok($im->add($mime), "message $i added"); if ($i == 4) { + $mark1 = $im->get_mark($im->{tip}); $im->remove($mime); + $mark2 = $im->get_mark($im->{tip}); } } if ('test remove later') { + $mark3 = $im->get_mark($im->{tip}); $mime->header_set('Message-Id', "<5\@example.com>"); $im->remove($mime); + $mark4 = $im->get_mark($im->{tip}); } $im->done; @@ -58,6 +60,7 @@ my $msgmap; $minmax = [ $ibx->mm->minmax ]; ok(defined $minmax->[0] && defined $minmax->[1], 'minmax defined'); is_deeply($minmax, [ 1, 10 ], 'minmax as expected'); + is($ibx->mm->num_highwater, 10, 'num_highwater as expected'); my ($min, $max) = @$minmax; $msgmap = $ibx->mm->msg_range(\$min, $max); @@ -82,11 +85,13 @@ my $msgmap; is($@, '', 'no error from reindexing'); $im->done; + is($ibx->mm->num_highwater, 10, 'num_highwater as expected'); + my ($min, $max) = $ibx->mm->minmax; is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged'); } -my $xap = "$mainrepo/public-inbox/xapian".PublicInbox::Search::SCHEMA_VERSION(); +my $xap = "$inboxdir/public-inbox/xapian".PublicInbox::Search::SCHEMA_VERSION(); remove_tree($xap); ok(!-d $xap, 'Xapian directories removed'); { @@ -102,12 +107,13 @@ ok(!-d $xap, 'Xapian directories removed'); delete $ibx->{mm}; is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged'); + is($ibx->mm->num_highwater, 10, 'num_highwater as expected'); my ($min, $max) = $ibx->mm->minmax; is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged'); } -ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap'); +ok(unlink "$inboxdir/public-inbox/msgmap.sqlite3", 'remove msgmap'); remove_tree($xap); ok(!-d $xap, 'Xapian directories removed again'); { @@ -124,12 +130,13 @@ ok(!-d $xap, 'Xapian directories removed again'); ok(-d $xap, 'Xapian directories recreated'); delete $ibx->{mm}; is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged'); + is($ibx->mm->num_highwater, 10, 'num_highwater as expected'); my ($min, $max) = $ibx->mm->minmax; is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged'); } -ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap'); +ok(unlink "$inboxdir/public-inbox/msgmap.sqlite3", 'remove msgmap'); remove_tree($xap); ok(!-d $xap, 'Xapian directories removed again'); { @@ -146,12 +153,13 @@ ok(!-d $xap, 'Xapian directories removed again'); ok(-d $xap, 'Xapian directories recreated'); delete $ibx->{mm}; is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged'); + is($ibx->mm->num_highwater, 10, 'num_highwater as expected'); my ($min, $max) = @$minmax; is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged'); } -ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap'); +ok(unlink "$inboxdir/public-inbox/msgmap.sqlite3", 'remove msgmap'); remove_tree($xap); ok(!-d $xap, 'Xapian directories removed again'); { @@ -169,6 +177,7 @@ ok(!-d $xap, 'Xapian directories removed again'); ok(-d $xap, 'Xapian directories recreated'); delete $ibx->{mm}; is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged'); + is($ibx->mm->num_highwater, 10, 'num_highwater as expected'); my $mset = $ibx->search->query('hello world', {mset=>1}); isnt($mset->size, 0, 'got Xapian search results'); @@ -176,7 +185,7 @@ ok(!-d $xap, 'Xapian directories removed again'); is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged'); } -ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap'); +ok(unlink "$inboxdir/public-inbox/msgmap.sqlite3", 'remove msgmap'); remove_tree($xap); ok(!-d $xap, 'Xapian directories removed again'); { @@ -194,8 +203,8 @@ ok(!-d $xap, 'Xapian directories removed again'); ok(-d $xap, 'Xapian directories recreated'); delete $ibx->{mm}; is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged'); - my $mset = $ibx->search->reopen->query('hello world', {mset=>1}); - is($mset->size, 0, "no Xapian search results"); + is($ibx->mm->num_highwater, 10, 'num_highwater as expected'); + isnt($ibx->search, 'no search for basic'); my ($min, $max) = $ibx->mm->minmax; is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged'); @@ -209,7 +218,7 @@ ok(!-d $xap, 'Xapian directories removed again'); my @warn; local $SIG{__WARN__} = sub { push @warn, @_ }; my %config = %$ibx_config; - $config{indexleve} = 'medium'; + $config{indexlevel} = 'medium'; my $ibx = PublicInbox::Inbox->new(\%config); my $rw = PublicInbox::SearchIdx->new($ibx, 1); eval { $rw->index_sync }; @@ -218,8 +227,206 @@ ok(!-d $xap, 'Xapian directories removed again'); my $mset = $ibx->search->reopen->query('hello world', {mset=>1}); isnt($mset->size, 0, 'search OK after basic -> medium'); + is($ibx->mm->num_highwater, 10, 'num_highwater as expected'); + my ($min, $max) = $ibx->mm->minmax; is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged'); } +# An incremental indexing test +ok(unlink "$inboxdir/public-inbox/msgmap.sqlite3", 'remove msgmap'); +remove_tree($xap); +ok(!-d $xap, 'Xapian directories removed again'); +{ + my @warn; + local $SIG{__WARN__} = sub { push @warn, @_ }; + my %config = %$ibx_config; + my $ibx = PublicInbox::Inbox->new(\%config); + my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx); + my $rw = PublicInbox::SearchIdx->new($ibx, 1); + # mark1 4 simple additions in the same index_sync + eval { $rw->index_sync({ref => $mark1}) }; + is($@, '', 'no error from reindexing without msgmap'); + is_deeply(\@warn, [], 'no warnings'); + $im->done; + my ($min, $max) = $ibx->mm->minmax; + is($min, 1, 'min as expected'); + is($max, 4, 'max as expected'); + is($ibx->mm->num_highwater, 4, 'num_highwater as expected'); + is_deeply($ibx->mm->msg_range(\$min, $max), + [ + [1, '1@example.com' ], + [2, '2@example.com' ], + [3, '3@example.com' ], + [4, '4@example.com' ], + ], 'msgmap as expected' ); +} +{ + my @warn; + local $SIG{__WARN__} = sub { push @warn, @_ }; + my %config = %$ibx_config; + my $ibx = PublicInbox::Inbox->new(\%config); + my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx); + my $rw = PublicInbox::SearchIdx->new($ibx, 1); + # mark2 A delete separated form and add in the same index_sync + eval { $rw->index_sync({ref => $mark2}) }; + is($@, '', 'no error from reindexing without msgmap'); + is_deeply(\@warn, [], 'no warnings'); + $im->done; + my ($min, $max) = $ibx->mm->minmax; + is($min, 1, 'min as expected'); + is($max, 3, 'max as expected'); + is($ibx->mm->num_highwater, 4, 'num_highwater as expected'); + is_deeply($ibx->mm->msg_range(\$min, $max), + [ + [1, '1@example.com' ], + [2, '2@example.com' ], + [3, '3@example.com' ], + ], 'msgmap as expected' ); +} +{ + my @warn; + local $SIG{__WARN__} = sub { push @warn, @_ }; + my %config = %$ibx_config; + my $ibx = PublicInbox::Inbox->new(\%config); + my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx); + my $rw = PublicInbox::SearchIdx->new($ibx, 1); + # mark3 adds following the delete at mark2 + eval { $rw->index_sync({ref => $mark3}) }; + is($@, '', 'no error from reindexing without msgmap'); + is_deeply(\@warn, [], 'no warnings'); + $im->done; + my ($min, $max) = $ibx->mm->minmax; + is($min, 1, 'min as expected'); + is($max, 10, 'max as expected'); + is($ibx->mm->num_highwater, 10, 'num_highwater as expected'); + is_deeply($ibx->mm->msg_range(\$min, $max), + [ + [1, '1@example.com' ], + [2, '2@example.com' ], + [3, '3@example.com' ], + [5, '5@example.com' ], + [6, '6@example.com' ], + [7, '7@example.com' ], + [8, '8@example.com' ], + [9, '9@example.com' ], + [10, '10@example.com' ], + ], 'msgmap as expected' ); +} +{ + my @warn; + local $SIG{__WARN__} = sub { push @warn, @_ }; + my %config = %$ibx_config; + my $ibx = PublicInbox::Inbox->new(\%config); + my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx); + my $rw = PublicInbox::SearchIdx->new($ibx, 1); + # mark4 A delete of an older message + eval { $rw->index_sync({ref => $mark4}) }; + is($@, '', 'no error from reindexing without msgmap'); + is_deeply(\@warn, [], 'no warnings'); + $im->done; + my ($min, $max) = $ibx->mm->minmax; + is($min, 1, 'min as expected'); + is($max, 10, 'max as expected'); + is($ibx->mm->num_highwater, 10, 'num_highwater as expected'); + is_deeply($ibx->mm->msg_range(\$min, $max), + [ + [1, '1@example.com' ], + [2, '2@example.com' ], + [3, '3@example.com' ], + [6, '6@example.com' ], + [7, '7@example.com' ], + [8, '8@example.com' ], + [9, '9@example.com' ], + [10, '10@example.com' ], + ], 'msgmap as expected' ); +} + + +# Another incremental indexing test +ok(unlink "$inboxdir/public-inbox/msgmap.sqlite3", 'remove msgmap'); +remove_tree($xap); +ok(!-d $xap, 'Xapian directories removed again'); +{ + my @warn; + local $SIG{__WARN__} = sub { push @warn, @_ }; + my %config = %$ibx_config; + my $ibx = PublicInbox::Inbox->new(\%config); + my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx); + my $rw = PublicInbox::SearchIdx->new($ibx, 1); + # mark2 an add and it's delete in the same index_sync + eval { $rw->index_sync({ref => $mark2}) }; + is($@, '', 'no error from reindexing without msgmap'); + is_deeply(\@warn, [], 'no warnings'); + $im->done; + my ($min, $max) = $ibx->mm->minmax; + is($min, 1, 'min as expected'); + is($max, 3, 'max as expected'); + is($ibx->mm->num_highwater, 4, 'num_highwater as expected'); + is_deeply($ibx->mm->msg_range(\$min, $max), + [ + [1, '1@example.com' ], + [2, '2@example.com' ], + [3, '3@example.com' ], + ], 'msgmap as expected' ); +} +{ + my @warn; + local $SIG{__WARN__} = sub { push @warn, @_ }; + my %config = %$ibx_config; + my $ibx = PublicInbox::Inbox->new(\%config); + my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx); + my $rw = PublicInbox::SearchIdx->new($ibx, 1); + # mark3 adds following the delete at mark2 + eval { $rw->index_sync({ref => $mark3}) }; + is($@, '', 'no error from reindexing without msgmap'); + is_deeply(\@warn, [], 'no warnings'); + $im->done; + my ($min, $max) = $ibx->mm->minmax; + is($min, 1, 'min as expected'); + is($max, 10, 'max as expected'); + is($ibx->mm->num_highwater, 10, 'num_highwater as expected'); + is_deeply($ibx->mm->msg_range(\$min, $max), + [ + [1, '1@example.com' ], + [2, '2@example.com' ], + [3, '3@example.com' ], + [5, '5@example.com' ], + [6, '6@example.com' ], + [7, '7@example.com' ], + [8, '8@example.com' ], + [9, '9@example.com' ], + [10, '10@example.com' ], + ], 'msgmap as expected' ); +} +{ + my @warn; + local $SIG{__WARN__} = sub { push @warn, @_ }; + my %config = %$ibx_config; + my $ibx = PublicInbox::Inbox->new(\%config); + my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx); + my $rw = PublicInbox::SearchIdx->new($ibx, 1); + # mark4 A delete of an older message + eval { $rw->index_sync({ref => $mark4}) }; + is($@, '', 'no error from reindexing without msgmap'); + is_deeply(\@warn, [], 'no warnings'); + $im->done; + my ($min, $max) = $ibx->mm->minmax; + is($min, 1, 'min as expected'); + is($max, 10, 'max as expected'); + is($ibx->mm->num_highwater, 10, 'num_highwater as expected'); + is_deeply($ibx->mm->msg_range(\$min, $max), + [ + [1, '1@example.com' ], + [2, '2@example.com' ], + [3, '3@example.com' ], + [6, '6@example.com' ], + [7, '7@example.com' ], + [8, '8@example.com' ], + [9, '9@example.com' ], + [10, '10@example.com' ], + ], 'msgmap as expected' ); +} + + done_testing();