1 # Copyright (C) 2018 all contributors <meta@public-inbox.org>
2 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
7 use PublicInbox::ContentId qw(content_digest);
8 use File::Temp qw/tempdir/;
9 use File::Path qw(remove_tree);
11 foreach my $mod (qw(DBD::SQLite Search::Xapian)) {
13 plan skip_all => "$mod missing for v2reindex.t" if $@;
15 use_ok 'PublicInbox::V2Writable';
16 my $mainrepo = tempdir('pi-v2reindex-XXXXXX', TMPDIR => 1, CLEANUP => 1);
18 mainrepo => $mainrepo,
19 name => 'test-v2writable',
21 -primary_address => 'test@example.com',
25 open my $fh, '<', 'COPYING' or die "can't open COPYING: $!";
29 $agpl or die "AGPL or die :P\n";
30 my $phrase = q("defending all users' freedom");
31 my $mime = PublicInbox::MIME->create(
33 From => 'a@example.com',
34 To => 'test@example.com',
35 Subject => 'this is a subject',
36 Date => 'Fri, 02 Oct 1993 00:00:00 +0000',
40 local $ENV{NPROC} = 2;
43 my ($mark1, $mark2, $mark3, $mark4);
45 my %config = %$ibx_config;
46 my $ibx = PublicInbox::Inbox->new(\%config);
47 my $im = PublicInbox::V2Writable->new($ibx, 1);
48 my $im0 = $im->importer();
49 foreach my $i (1..10) {
50 $mime->header_set('Message-Id', "<$i\@example.com>");
51 ok($im->add($mime), "message $i added");
53 $mark1 = $im0->get_mark($im0->{tip});
55 $mark2 = $im0->get_mark($im0->{tip});
59 if ('test remove later') {
60 $mark3 = $im0->get_mark($im0->{tip});
61 $mime->header_set('Message-Id', "<5\@example.com>");
63 $mark4 = $im0->get_mark($im0->{tip});
67 $minmax = [ $ibx->mm->minmax ];
68 ok(defined $minmax->[0] && defined $minmax->[1], 'minmax defined');
69 is_deeply($minmax, [ 1, 10 ], 'minmax as expected');
70 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
72 my ($min, $max) = @$minmax;
73 $msgmap = $ibx->mm->msg_range(\$min, $max);
75 [1, '1@example.com' ],
76 [2, '2@example.com' ],
77 [3, '3@example.com' ],
78 [6, '6@example.com' ],
79 [7, '7@example.com' ],
80 [8, '8@example.com' ],
81 [9, '9@example.com' ],
82 [10, '10@example.com' ],
83 ], 'msgmap as expected');
87 my %config = %$ibx_config;
88 my $ibx = PublicInbox::Inbox->new(\%config);
89 my $im = PublicInbox::V2Writable->new($ibx, 1);
90 eval { $im->index_sync({reindex => 1}) };
91 is($@, '', 'no error from reindexing');
95 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
96 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
98 my ($min, $max) = $ibx->mm->minmax;
99 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
102 my $xap = "$mainrepo/xap".PublicInbox::Search::SCHEMA_VERSION();
104 ok(!-d $xap, 'Xapian directories removed');
106 my %config = %$ibx_config;
107 my $ibx = PublicInbox::Inbox->new(\%config);
108 my $im = PublicInbox::V2Writable->new($ibx, 1);
109 eval { $im->index_sync({reindex => 1}) };
110 is($@, '', 'no error from reindexing');
112 ok(-d $xap, 'Xapian directories recreated');
115 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
116 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
118 my ($min, $max) = $ibx->mm->minmax;
119 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
122 ok(unlink "$mainrepo/msgmap.sqlite3", 'remove msgmap');
124 ok(!-d $xap, 'Xapian directories removed again');
127 local $SIG{__WARN__} = sub { push @warn, @_ };
128 my %config = %$ibx_config;
129 my $ibx = PublicInbox::Inbox->new(\%config);
130 my $im = PublicInbox::V2Writable->new($ibx, 1);
131 eval { $im->index_sync({reindex => 1}) };
132 is($@, '', 'no error from reindexing without msgmap');
133 is(scalar(@warn), 0, 'no warnings from reindexing');
135 ok(-d $xap, 'Xapian directories recreated');
137 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
138 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
140 my ($min, $max) = $ibx->mm->minmax;
141 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
145 ok(unlink "$mainrepo/msgmap.sqlite3", 'remove msgmap');
147 ok(!-d $xap, 'Xapian directories removed again');
150 local $SIG{__WARN__} = sub { push @warn, @_ };
151 my %config = %$ibx_config;
152 my $ibx = PublicInbox::Inbox->new(\%config);
153 my $im = PublicInbox::V2Writable->new($ibx, 1);
154 eval { $im->index_sync({reindex => 1}) };
155 is($@, '', 'no error from reindexing without msgmap');
156 is_deeply(\@warn, [], 'no warnings');
158 ok(-d $xap, 'Xapian directories recreated');
160 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
161 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
162 my $mset = $ibx->search->query($phrase, {mset=>1});
163 isnt($mset->size, 0, "phrase search succeeds on indexlevel=full");
164 for (<"$xap/*/*">) { $sizes{$ibx->{indexlevel}} += -s _ if -f $_ }
166 my ($min, $max) = $ibx->mm->minmax;
167 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
170 ok(unlink "$mainrepo/msgmap.sqlite3", 'remove msgmap');
172 ok(!-d $xap, 'Xapian directories removed again');
175 local $SIG{__WARN__} = sub { push @warn, @_ };
176 my %config = %$ibx_config;
177 $config{indexlevel} = 'medium';
178 my $ibx = PublicInbox::Inbox->new(\%config);
179 my $im = PublicInbox::V2Writable->new($ibx);
180 eval { $im->index_sync({reindex => 1}) };
181 is($@, '', 'no error from reindexing without msgmap');
182 is_deeply(\@warn, [], 'no warnings');
184 ok(-d $xap, 'Xapian directories recreated');
186 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
187 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
190 # not sure why, but Xapian seems to fallback to terms and
191 # phrase searches still work
192 delete $ibx->{search};
193 my $mset = $ibx->search->query($phrase, {mset=>1});
194 is($mset->size, 0, 'phrase search does not work on medium');
198 my $mset = $ibx->search->query($words, {mset=>1});
199 isnt($mset->size, 0, "normal search works on indexlevel=medium");
200 for (<"$xap/*/*">) { $sizes{$ibx->{indexlevel}} += -s _ if -f $_ }
202 ok($sizes{full} > $sizes{medium}, 'medium is smaller than full');
205 my ($min, $max) = $ibx->mm->minmax;
206 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
209 ok(unlink "$mainrepo/msgmap.sqlite3", 'remove msgmap');
211 ok(!-d $xap, 'Xapian directories removed again');
214 local $SIG{__WARN__} = sub { push @warn, @_ };
215 my %config = %$ibx_config;
216 $config{indexlevel} = 'basic';
217 my $ibx = PublicInbox::Inbox->new(\%config);
218 my $im = PublicInbox::V2Writable->new($ibx);
219 eval { $im->index_sync({reindex => 1}) };
220 is($@, '', 'no error from reindexing without msgmap');
221 is_deeply(\@warn, [], 'no warnings');
223 ok(-d $xap, 'Xapian directories recreated');
225 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
226 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
227 my $mset = $ibx->search->query('freedom', {mset=>1});
228 is($mset->size, 0, "search fails on indexlevel='basic'");
229 for (<"$xap/*/*">) { $sizes{$ibx->{indexlevel}} += -s _ if -f $_ }
230 ok($sizes{medium} > $sizes{basic}, 'basic is smaller than medium');
232 my ($min, $max) = $ibx->mm->minmax;
233 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
237 # An incremental indexing test
238 ok(unlink "$mainrepo/msgmap.sqlite3", 'remove msgmap');
240 ok(!-d $xap, 'Xapian directories removed again');
243 local $SIG{__WARN__} = sub { push @warn, @_ };
244 my %config = %$ibx_config;
245 my $ibx = PublicInbox::Inbox->new(\%config);
246 # mark1 4 simple additions in the same index_sync
247 $ibx->{ref_head} = $mark1;
248 my $im = PublicInbox::V2Writable->new($ibx);
249 eval { $im->index_sync() };
250 is($@, '', 'no error from reindexing without msgmap');
251 is_deeply(\@warn, [], 'no warnings');
253 my ($min, $max) = $ibx->mm->minmax;
254 is($min, 1, 'min as expected');
255 is($max, 4, 'max as expected');
256 is($ibx->mm->num_highwater, 4, 'num_highwater as expected');
257 is_deeply($ibx->mm->msg_range(\$min, $max),
259 [1, '1@example.com' ],
260 [2, '2@example.com' ],
261 [3, '3@example.com' ],
262 [4, '4@example.com' ],
263 ], 'msgmap as expected' );
267 local $SIG{__WARN__} = sub { push @warn, @_ };
268 my %config = %$ibx_config;
269 my $ibx = PublicInbox::Inbox->new(\%config);
270 # mark2 A delete separated from an add in the same index_sync
271 $ibx->{ref_head} = $mark2;
272 my $im = PublicInbox::V2Writable->new($ibx);
273 eval { $im->index_sync() };
274 is($@, '', 'no error from reindexing without msgmap');
275 is_deeply(\@warn, [], 'no warnings');
277 my ($min, $max) = $ibx->mm->minmax;
278 is($min, 1, 'min as expected');
279 is($max, 3, 'max as expected');
280 is($ibx->mm->num_highwater, 4, 'num_highwater as expected');
281 is_deeply($ibx->mm->msg_range(\$min, $max),
283 [1, '1@example.com' ],
284 [2, '2@example.com' ],
285 [3, '3@example.com' ],
286 ], 'msgmap as expected' );
290 local $SIG{__WARN__} = sub { push @warn, @_ };
291 my %config = %$ibx_config;
292 my $ibx = PublicInbox::Inbox->new(\%config);
293 # mark3 adds following the delete at mark2
294 $ibx->{ref_head} = $mark3;
295 my $im = PublicInbox::V2Writable->new($ibx);
296 eval { $im->index_sync() };
297 is($@, '', 'no error from reindexing without msgmap');
298 is_deeply(\@warn, [], 'no warnings');
300 my ($min, $max) = $ibx->mm->minmax;
301 is($min, 1, 'min as expected');
302 is($max, 10, 'max as expected');
303 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
304 is_deeply($ibx->mm->msg_range(\$min, $max),
306 [1, '1@example.com' ],
307 [2, '2@example.com' ],
308 [3, '3@example.com' ],
309 [5, '5@example.com' ],
310 [6, '6@example.com' ],
311 [7, '7@example.com' ],
312 [8, '8@example.com' ],
313 [9, '9@example.com' ],
314 [10, '10@example.com' ],
315 ], 'msgmap as expected' );
319 local $SIG{__WARN__} = sub { push @warn, @_ };
320 my %config = %$ibx_config;
321 my $ibx = PublicInbox::Inbox->new(\%config);
322 # mark4 A delete of an older message
323 $ibx->{ref_head} = $mark4;
324 my $im = PublicInbox::V2Writable->new($ibx);
325 eval { $im->index_sync() };
326 is($@, '', 'no error from reindexing without msgmap');
327 is_deeply(\@warn, [], 'no warnings');
329 my ($min, $max) = $ibx->mm->minmax;
330 is($min, 1, 'min as expected');
331 is($max, 10, 'max as expected');
332 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
333 is_deeply($ibx->mm->msg_range(\$min, $max),
335 [1, '1@example.com' ],
336 [2, '2@example.com' ],
337 [3, '3@example.com' ],
338 [6, '6@example.com' ],
339 [7, '7@example.com' ],
340 [8, '8@example.com' ],
341 [9, '9@example.com' ],
342 [10, '10@example.com' ],
343 ], 'msgmap as expected' );
347 # Another incremental indexing test
348 ok(unlink "$mainrepo/msgmap.sqlite3", 'remove msgmap');
350 ok(!-d $xap, 'Xapian directories removed again');
353 local $SIG{__WARN__} = sub { push @warn, @_ };
354 my %config = %$ibx_config;
355 my $ibx = PublicInbox::Inbox->new(\%config);
356 # mark2 an add and it's delete in the same index_sync
357 $ibx->{ref_head} = $mark2;
358 my $im = PublicInbox::V2Writable->new($ibx);
359 eval { $im->index_sync() };
360 is($@, '', 'no error from reindexing without msgmap');
361 is_deeply(\@warn, [], 'no warnings');
363 my ($min, $max) = $ibx->mm->minmax;
364 is($min, 1, 'min as expected');
365 is($max, 3, 'max as expected');
366 is($ibx->mm->num_highwater, 4, 'num_highwater as expected');
367 is_deeply($ibx->mm->msg_range(\$min, $max),
369 [1, '1@example.com' ],
370 [2, '2@example.com' ],
371 [3, '3@example.com' ],
372 ], 'msgmap as expected' );
376 local $SIG{__WARN__} = sub { push @warn, @_ };
377 my %config = %$ibx_config;
378 my $ibx = PublicInbox::Inbox->new(\%config);
379 # mark3 adds following the delete at mark2
380 $ibx->{ref_head} = $mark3;
381 my $im = PublicInbox::V2Writable->new($ibx);
382 eval { $im->index_sync() };
383 is($@, '', 'no error from reindexing without msgmap');
384 is_deeply(\@warn, [], 'no warnings');
386 my ($min, $max) = $ibx->mm->minmax;
387 is($min, 1, 'min as expected');
388 is($max, 10, 'max as expected');
389 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
390 is_deeply($ibx->mm->msg_range(\$min, $max),
392 [1, '1@example.com' ],
393 [2, '2@example.com' ],
394 [3, '3@example.com' ],
395 [5, '5@example.com' ],
396 [6, '6@example.com' ],
397 [7, '7@example.com' ],
398 [8, '8@example.com' ],
399 [9, '9@example.com' ],
400 [10, '10@example.com' ],
401 ], 'msgmap as expected' );
405 local $SIG{__WARN__} = sub { push @warn, @_ };
406 my %config = %$ibx_config;
407 my $ibx = PublicInbox::Inbox->new(\%config);
408 # mark4 A delete of an older message
409 $ibx->{ref_head} = $mark4;
410 my $im = PublicInbox::V2Writable->new($ibx);
411 eval { $im->index_sync() };
412 is($@, '', 'no error from reindexing without msgmap');
413 is_deeply(\@warn, [], 'no warnings');
415 my ($min, $max) = $ibx->mm->minmax;
416 is($min, 1, 'min as expected');
417 is($max, 10, 'max as expected');
418 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
419 is_deeply($ibx->mm->msg_range(\$min, $max),
421 [1, '1@example.com' ],
422 [2, '2@example.com' ],
423 [3, '3@example.com' ],
424 [6, '6@example.com' ],
425 [7, '7@example.com' ],
426 [8, '8@example.com' ],
427 [9, '9@example.com' ],
428 [10, '10@example.com' ],
429 ], 'msgmap as expected' );