1 # Copyright (C) 2018 all contributors <meta@public-inbox.org>
2 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
7 use PublicInbox::ContentId qw(content_digest);
8 use File::Temp qw/tempdir/;
9 use File::Path qw(remove_tree);
10 require './t/common.perl';
13 foreach my $mod (qw(DBD::SQLite Search::Xapian)) {
15 plan skip_all => "$mod missing for v2reindex.t" if $@;
17 use_ok 'PublicInbox::V2Writable';
18 my $mainrepo = tempdir('pi-v2reindex-XXXXXX', TMPDIR => 1, CLEANUP => 1);
20 mainrepo => $mainrepo,
21 name => 'test-v2writable',
23 -primary_address => 'test@example.com',
27 open my $fh, '<', 'COPYING' or die "can't open COPYING: $!";
31 $agpl or die "AGPL or die :P\n";
32 my $phrase = q("defending all users' freedom");
33 my $mime = PublicInbox::MIME->create(
35 From => 'a@example.com',
36 To => 'test@example.com',
37 Subject => 'this is a subject',
38 Date => 'Fri, 02 Oct 1993 00:00:00 +0000',
42 local $ENV{NPROC} = 2;
45 my ($mark1, $mark2, $mark3, $mark4);
47 my %config = %$ibx_config;
48 my $ibx = PublicInbox::Inbox->new(\%config);
49 my $im = PublicInbox::V2Writable->new($ibx, 1);
50 my $im0 = $im->importer();
51 foreach my $i (1..10) {
52 $mime->header_set('Message-Id', "<$i\@example.com>");
53 ok($im->add($mime), "message $i added");
55 $mark1 = $im0->get_mark($im0->{tip});
57 $mark2 = $im0->get_mark($im0->{tip});
61 if ('test remove later') {
62 $mark3 = $im0->get_mark($im0->{tip});
63 $mime->header_set('Message-Id', "<5\@example.com>");
65 $mark4 = $im0->get_mark($im0->{tip});
69 $minmax = [ $ibx->mm->minmax ];
70 ok(defined $minmax->[0] && defined $minmax->[1], 'minmax defined');
71 is_deeply($minmax, [ 1, 10 ], 'minmax as expected');
72 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
74 my ($min, $max) = @$minmax;
75 $msgmap = $ibx->mm->msg_range(\$min, $max);
77 [1, '1@example.com' ],
78 [2, '2@example.com' ],
79 [3, '3@example.com' ],
80 [6, '6@example.com' ],
81 [7, '7@example.com' ],
82 [8, '8@example.com' ],
83 [9, '9@example.com' ],
84 [10, '10@example.com' ],
85 ], 'msgmap as expected');
89 my %config = %$ibx_config;
90 my $ibx = PublicInbox::Inbox->new(\%config);
91 my $im = PublicInbox::V2Writable->new($ibx, 1);
92 eval { $im->index_sync({reindex => 1}) };
93 is($@, '', 'no error from reindexing');
97 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
98 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
100 my ($min, $max) = $ibx->mm->minmax;
101 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
104 my $xap = "$mainrepo/xap".PublicInbox::Search::SCHEMA_VERSION();
106 ok(!-d $xap, 'Xapian directories removed');
108 my %config = %$ibx_config;
109 my $ibx = PublicInbox::Inbox->new(\%config);
110 my $im = PublicInbox::V2Writable->new($ibx, 1);
111 eval { $im->index_sync({reindex => 1}) };
112 is($@, '', 'no error from reindexing');
114 ok(-d $xap, 'Xapian directories recreated');
117 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
118 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
120 my ($min, $max) = $ibx->mm->minmax;
121 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
124 ok(unlink "$mainrepo/msgmap.sqlite3", 'remove msgmap');
126 ok(!-d $xap, 'Xapian directories removed again');
129 local $SIG{__WARN__} = sub { push @warn, @_ };
130 my %config = %$ibx_config;
131 my $ibx = PublicInbox::Inbox->new(\%config);
132 my $im = PublicInbox::V2Writable->new($ibx, 1);
133 eval { $im->index_sync({reindex => 1}) };
134 is($@, '', 'no error from reindexing without msgmap');
135 is(scalar(@warn), 0, 'no warnings from reindexing');
137 ok(-d $xap, 'Xapian directories recreated');
139 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
140 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
142 my ($min, $max) = $ibx->mm->minmax;
143 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
147 ok(unlink "$mainrepo/msgmap.sqlite3", 'remove msgmap');
149 ok(!-d $xap, 'Xapian directories removed again');
152 local $SIG{__WARN__} = sub { push @warn, @_ };
153 my %config = %$ibx_config;
154 my $ibx = PublicInbox::Inbox->new(\%config);
155 my $im = PublicInbox::V2Writable->new($ibx, 1);
156 eval { $im->index_sync({reindex => 1}) };
157 is($@, '', 'no error from reindexing without msgmap');
158 is_deeply(\@warn, [], 'no warnings');
160 ok(-d $xap, 'Xapian directories recreated');
162 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
163 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
164 my $mset = $ibx->search->query($phrase, {mset=>1});
165 isnt($mset->size, 0, "phrase search succeeds on indexlevel=full");
166 for (<"$xap/*/*">) { $sizes{$ibx->{indexlevel}} += -s _ if -f $_ }
168 my ($min, $max) = $ibx->mm->minmax;
169 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
172 ok(unlink "$mainrepo/msgmap.sqlite3", 'remove msgmap');
174 ok(!-d $xap, 'Xapian directories removed again');
177 local $SIG{__WARN__} = sub { push @warn, @_ };
178 my %config = %$ibx_config;
179 $config{indexlevel} = 'medium';
180 my $ibx = PublicInbox::Inbox->new(\%config);
181 my $im = PublicInbox::V2Writable->new($ibx);
182 eval { $im->index_sync({reindex => 1}) };
183 is($@, '', 'no error from reindexing without msgmap');
184 is_deeply(\@warn, [], 'no warnings');
186 ok(-d $xap, 'Xapian directories recreated');
188 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
189 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
192 # not sure why, but Xapian seems to fallback to terms and
193 # phrase searches still work
194 delete $ibx->{search};
195 my $mset = $ibx->search->query($phrase, {mset=>1});
196 is($mset->size, 0, 'phrase search does not work on medium');
200 my $mset = $ibx->search->query($words, {mset=>1});
201 isnt($mset->size, 0, "normal search works on indexlevel=medium");
202 for (<"$xap/*/*">) { $sizes{$ibx->{indexlevel}} += -s _ if -f $_ }
204 ok($sizes{full} > $sizes{medium}, 'medium is smaller than full');
207 my ($min, $max) = $ibx->mm->minmax;
208 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
211 ok(unlink "$mainrepo/msgmap.sqlite3", 'remove msgmap');
213 ok(!-d $xap, 'Xapian directories removed again');
216 local $SIG{__WARN__} = sub { push @warn, @_ };
217 my %config = %$ibx_config;
218 $config{indexlevel} = 'basic';
219 my $ibx = PublicInbox::Inbox->new(\%config);
220 my $im = PublicInbox::V2Writable->new($ibx);
221 eval { $im->index_sync({reindex => 1}) };
222 is($@, '', 'no error from reindexing without msgmap');
223 is_deeply(\@warn, [], 'no warnings');
225 ok(-d $xap, 'Xapian directories recreated');
227 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
228 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
229 my $mset = $ibx->search->query('freedom', {mset=>1});
230 is($mset->size, 0, "search fails on indexlevel='basic'");
231 for (<"$xap/*/*">) { $sizes{$ibx->{indexlevel}} += -s _ if -f $_ }
232 ok($sizes{medium} > $sizes{basic}, 'basic is smaller than medium');
234 my ($min, $max) = $ibx->mm->minmax;
235 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
239 # An incremental indexing test
240 ok(unlink "$mainrepo/msgmap.sqlite3", 'remove msgmap');
242 ok(!-d $xap, 'Xapian directories removed again');
245 local $SIG{__WARN__} = sub { push @warn, @_ };
246 my %config = %$ibx_config;
247 my $ibx = PublicInbox::Inbox->new(\%config);
248 # mark1 4 simple additions in the same index_sync
249 $ibx->{ref_head} = $mark1;
250 my $im = PublicInbox::V2Writable->new($ibx);
251 eval { $im->index_sync() };
252 is($@, '', 'no error from reindexing without msgmap');
253 is_deeply(\@warn, [], 'no warnings');
255 my ($min, $max) = $ibx->mm->minmax;
256 is($min, 1, 'min as expected');
257 is($max, 4, 'max as expected');
258 is($ibx->mm->num_highwater, 4, 'num_highwater as expected');
259 is_deeply($ibx->mm->msg_range(\$min, $max),
261 [1, '1@example.com' ],
262 [2, '2@example.com' ],
263 [3, '3@example.com' ],
264 [4, '4@example.com' ],
265 ], 'msgmap as expected' );
269 local $SIG{__WARN__} = sub { push @warn, @_ };
270 my %config = %$ibx_config;
271 my $ibx = PublicInbox::Inbox->new(\%config);
272 # mark2 A delete separated from an add in the same index_sync
273 $ibx->{ref_head} = $mark2;
274 my $im = PublicInbox::V2Writable->new($ibx);
275 eval { $im->index_sync() };
276 is($@, '', 'no error from reindexing without msgmap');
277 is_deeply(\@warn, [], 'no warnings');
279 my ($min, $max) = $ibx->mm->minmax;
280 is($min, 1, 'min as expected');
281 is($max, 3, 'max as expected');
282 is($ibx->mm->num_highwater, 4, 'num_highwater as expected');
283 is_deeply($ibx->mm->msg_range(\$min, $max),
285 [1, '1@example.com' ],
286 [2, '2@example.com' ],
287 [3, '3@example.com' ],
288 ], 'msgmap as expected' );
292 local $SIG{__WARN__} = sub { push @warn, @_ };
293 my %config = %$ibx_config;
294 my $ibx = PublicInbox::Inbox->new(\%config);
295 # mark3 adds following the delete at mark2
296 $ibx->{ref_head} = $mark3;
297 my $im = PublicInbox::V2Writable->new($ibx);
298 eval { $im->index_sync() };
299 is($@, '', 'no error from reindexing without msgmap');
300 is_deeply(\@warn, [], 'no warnings');
302 my ($min, $max) = $ibx->mm->minmax;
303 is($min, 1, 'min as expected');
304 is($max, 10, 'max as expected');
305 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
306 is_deeply($ibx->mm->msg_range(\$min, $max),
308 [1, '1@example.com' ],
309 [2, '2@example.com' ],
310 [3, '3@example.com' ],
311 [5, '5@example.com' ],
312 [6, '6@example.com' ],
313 [7, '7@example.com' ],
314 [8, '8@example.com' ],
315 [9, '9@example.com' ],
316 [10, '10@example.com' ],
317 ], 'msgmap as expected' );
321 local $SIG{__WARN__} = sub { push @warn, @_ };
322 my %config = %$ibx_config;
323 my $ibx = PublicInbox::Inbox->new(\%config);
324 # mark4 A delete of an older message
325 $ibx->{ref_head} = $mark4;
326 my $im = PublicInbox::V2Writable->new($ibx);
327 eval { $im->index_sync() };
328 is($@, '', 'no error from reindexing without msgmap');
329 is_deeply(\@warn, [], 'no warnings');
331 my ($min, $max) = $ibx->mm->minmax;
332 is($min, 1, 'min as expected');
333 is($max, 10, 'max as expected');
334 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
335 is_deeply($ibx->mm->msg_range(\$min, $max),
337 [1, '1@example.com' ],
338 [2, '2@example.com' ],
339 [3, '3@example.com' ],
340 [6, '6@example.com' ],
341 [7, '7@example.com' ],
342 [8, '8@example.com' ],
343 [9, '9@example.com' ],
344 [10, '10@example.com' ],
345 ], 'msgmap as expected' );
349 # Another incremental indexing test
350 ok(unlink "$mainrepo/msgmap.sqlite3", 'remove msgmap');
352 ok(!-d $xap, 'Xapian directories removed again');
355 local $SIG{__WARN__} = sub { push @warn, @_ };
356 my %config = %$ibx_config;
357 my $ibx = PublicInbox::Inbox->new(\%config);
358 # mark2 an add and it's delete in the same index_sync
359 $ibx->{ref_head} = $mark2;
360 my $im = PublicInbox::V2Writable->new($ibx);
361 eval { $im->index_sync() };
362 is($@, '', 'no error from reindexing without msgmap');
363 is_deeply(\@warn, [], 'no warnings');
365 my ($min, $max) = $ibx->mm->minmax;
366 is($min, 1, 'min as expected');
367 is($max, 3, 'max as expected');
368 is($ibx->mm->num_highwater, 4, 'num_highwater as expected');
369 is_deeply($ibx->mm->msg_range(\$min, $max),
371 [1, '1@example.com' ],
372 [2, '2@example.com' ],
373 [3, '3@example.com' ],
374 ], 'msgmap as expected' );
378 local $SIG{__WARN__} = sub { push @warn, @_ };
379 my %config = %$ibx_config;
380 my $ibx = PublicInbox::Inbox->new(\%config);
381 # mark3 adds following the delete at mark2
382 $ibx->{ref_head} = $mark3;
383 my $im = PublicInbox::V2Writable->new($ibx);
384 eval { $im->index_sync() };
385 is($@, '', 'no error from reindexing without msgmap');
386 is_deeply(\@warn, [], 'no warnings');
388 my ($min, $max) = $ibx->mm->minmax;
389 is($min, 1, 'min as expected');
390 is($max, 10, 'max as expected');
391 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
392 is_deeply($ibx->mm->msg_range(\$min, $max),
394 [1, '1@example.com' ],
395 [2, '2@example.com' ],
396 [3, '3@example.com' ],
397 [5, '5@example.com' ],
398 [6, '6@example.com' ],
399 [7, '7@example.com' ],
400 [8, '8@example.com' ],
401 [9, '9@example.com' ],
402 [10, '10@example.com' ],
403 ], 'msgmap as expected' );
407 local $SIG{__WARN__} = sub { push @warn, @_ };
408 my %config = %$ibx_config;
409 my $ibx = PublicInbox::Inbox->new(\%config);
410 # mark4 A delete of an older message
411 $ibx->{ref_head} = $mark4;
412 my $im = PublicInbox::V2Writable->new($ibx);
413 eval { $im->index_sync() };
414 is($@, '', 'no error from reindexing without msgmap');
415 is_deeply(\@warn, [], 'no warnings');
417 my ($min, $max) = $ibx->mm->minmax;
418 is($min, 1, 'min as expected');
419 is($max, 10, 'max as expected');
420 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
421 is_deeply($ibx->mm->msg_range(\$min, $max),
423 [1, '1@example.com' ],
424 [2, '2@example.com' ],
425 [3, '3@example.com' ],
426 [6, '6@example.com' ],
427 [7, '7@example.com' ],
428 [8, '8@example.com' ],
429 [9, '9@example.com' ],
430 [10, '10@example.com' ],
431 ], 'msgmap as expected' );