1 # Copyright (C) 2018-2019 all contributors <meta@public-inbox.org>
2 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
7 use PublicInbox::ContentId qw(content_digest);
8 use File::Temp qw/tempdir/;
9 use File::Path qw(remove_tree);
10 require './t/common.perl';
13 foreach my $mod (qw(DBD::SQLite Search::Xapian)) {
15 plan skip_all => "$mod missing for v2reindex.t" if $@;
17 use_ok 'PublicInbox::V2Writable';
18 my $inboxdir = tempdir('pi-v2reindex-XXXXXX', TMPDIR => 1, CLEANUP => 1);
20 inboxdir => $inboxdir,
21 name => 'test-v2writable',
23 -primary_address => 'test@example.com',
27 open my $fh, '<', 'COPYING' or die "can't open COPYING: $!";
31 $agpl or die "AGPL or die :P\n";
32 my $phrase = q("defending all users' freedom");
33 my $mime = PublicInbox::MIME->create(
35 From => 'a@example.com',
36 To => 'test@example.com',
37 Subject => 'this is a subject',
38 Date => 'Fri, 02 Oct 1993 00:00:00 +0000',
44 my ($mark1, $mark2, $mark3, $mark4);
46 my %config = %$ibx_config;
47 my $ibx = PublicInbox::Inbox->new(\%config);
48 my $im = PublicInbox::V2Writable->new($ibx, {nproc => 1});
49 my $im0 = $im->importer();
50 foreach my $i (1..10) {
51 $mime->header_set('Message-Id', "<$i\@example.com>");
52 ok($im->add($mime), "message $i added");
54 $mark1 = $im0->get_mark($im0->{tip});
56 $mark2 = $im0->get_mark($im0->{tip});
60 if ('test remove later') {
61 $mark3 = $im0->get_mark($im0->{tip});
62 $mime->header_set('Message-Id', "<5\@example.com>");
64 $mark4 = $im0->get_mark($im0->{tip});
68 $minmax = [ $ibx->mm->minmax ];
69 ok(defined $minmax->[0] && defined $minmax->[1], 'minmax defined');
70 is_deeply($minmax, [ 1, 10 ], 'minmax as expected');
71 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
73 my ($min, $max) = @$minmax;
74 $msgmap = $ibx->mm->msg_range(\$min, $max);
76 [1, '1@example.com' ],
77 [2, '2@example.com' ],
78 [3, '3@example.com' ],
79 [6, '6@example.com' ],
80 [7, '7@example.com' ],
81 [8, '8@example.com' ],
82 [9, '9@example.com' ],
83 [10, '10@example.com' ],
84 ], 'msgmap as expected');
88 my %config = %$ibx_config;
89 my $ibx = PublicInbox::Inbox->new(\%config);
90 my $im = PublicInbox::V2Writable->new($ibx, 1);
91 eval { $im->index_sync({reindex => 1}) };
92 is($@, '', 'no error from reindexing');
96 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
97 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
99 my ($min, $max) = $ibx->mm->minmax;
100 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
103 my $xap = "$inboxdir/xap".PublicInbox::Search::SCHEMA_VERSION();
105 ok(!-d $xap, 'Xapian directories removed');
107 my %config = %$ibx_config;
108 my $ibx = PublicInbox::Inbox->new(\%config);
109 my $im = PublicInbox::V2Writable->new($ibx, 1);
110 eval { $im->index_sync({reindex => 1}) };
111 is($@, '', 'no error from reindexing');
113 ok(-d $xap, 'Xapian directories recreated');
116 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
117 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
119 my ($min, $max) = $ibx->mm->minmax;
120 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
123 ok(unlink "$inboxdir/msgmap.sqlite3", 'remove msgmap');
125 ok(!-d $xap, 'Xapian directories removed again');
128 local $SIG{__WARN__} = sub { push @warn, @_ };
129 my %config = %$ibx_config;
130 my $ibx = PublicInbox::Inbox->new(\%config);
131 my $im = PublicInbox::V2Writable->new($ibx, 1);
132 eval { $im->index_sync({reindex => 1}) };
133 is($@, '', 'no error from reindexing without msgmap');
134 is(scalar(@warn), 0, 'no warnings from reindexing');
136 ok(-d $xap, 'Xapian directories recreated');
138 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
139 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
141 my ($min, $max) = $ibx->mm->minmax;
142 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
146 ok(unlink "$inboxdir/msgmap.sqlite3", 'remove msgmap');
148 ok(!-d $xap, 'Xapian directories removed again');
151 local $SIG{__WARN__} = sub { push @warn, @_ };
152 my %config = %$ibx_config;
153 my $ibx = PublicInbox::Inbox->new(\%config);
154 my $im = PublicInbox::V2Writable->new($ibx, 1);
155 eval { $im->index_sync({reindex => 1}) };
156 is($@, '', 'no error from reindexing without msgmap');
157 is_deeply(\@warn, [], 'no warnings');
159 ok(-d $xap, 'Xapian directories recreated');
161 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
162 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
163 my $mset = $ibx->search->query($phrase, {mset=>1});
164 isnt($mset->size, 0, "phrase search succeeds on indexlevel=full");
165 for (<"$xap/*/*">) { $sizes{$ibx->{indexlevel}} += -s _ if -f $_ }
167 my ($min, $max) = $ibx->mm->minmax;
168 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
171 ok(unlink "$inboxdir/msgmap.sqlite3", 'remove msgmap');
173 ok(!-d $xap, 'Xapian directories removed again');
176 local $SIG{__WARN__} = sub { push @warn, @_ };
177 my %config = %$ibx_config;
178 $config{indexlevel} = 'medium';
179 my $ibx = PublicInbox::Inbox->new(\%config);
180 my $im = PublicInbox::V2Writable->new($ibx);
181 eval { $im->index_sync({reindex => 1}) };
182 is($@, '', 'no error from reindexing without msgmap');
183 is_deeply(\@warn, [], 'no warnings');
185 ok(-d $xap, 'Xapian directories recreated');
187 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
188 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
191 # not sure why, but Xapian seems to fallback to terms and
192 # phrase searches still work
193 delete $ibx->{search};
194 my $mset = $ibx->search->query($phrase, {mset=>1});
195 is($mset->size, 0, 'phrase search does not work on medium');
199 my $mset = $ibx->search->query($words, {mset=>1});
200 isnt($mset->size, 0, "normal search works on indexlevel=medium");
201 for (<"$xap/*/*">) { $sizes{$ibx->{indexlevel}} += -s _ if -f $_ }
203 ok($sizes{full} > $sizes{medium}, 'medium is smaller than full');
206 my ($min, $max) = $ibx->mm->minmax;
207 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
210 ok(unlink "$inboxdir/msgmap.sqlite3", 'remove msgmap');
212 ok(!-d $xap, 'Xapian directories removed again');
215 local $SIG{__WARN__} = sub { push @warn, @_ };
216 my %config = %$ibx_config;
217 $config{indexlevel} = 'basic';
218 my $ibx = PublicInbox::Inbox->new(\%config);
219 my $im = PublicInbox::V2Writable->new($ibx);
220 eval { $im->index_sync({reindex => 1}) };
221 is($@, '', 'no error from reindexing without msgmap');
222 is_deeply(\@warn, [], 'no warnings');
224 ok(-d $xap, 'Xapian directories recreated');
226 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
227 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
229 isnt($ibx->search, 'no search for basic');
231 for (<"$xap/*/*">) { $sizes{$ibx->{indexlevel}} += -s _ if -f $_ }
232 ok($sizes{medium} > $sizes{basic}, 'basic is smaller than medium');
234 my ($min, $max) = $ibx->mm->minmax;
235 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
239 # An incremental indexing test
240 ok(unlink "$inboxdir/msgmap.sqlite3", 'remove msgmap');
242 ok(!-d $xap, 'Xapian directories removed again');
245 local $SIG{__WARN__} = sub { push @warn, @_ };
246 my %config = %$ibx_config;
247 my $ibx = PublicInbox::Inbox->new(\%config);
248 # mark1 4 simple additions in the same index_sync
249 $ibx->{ref_head} = $mark1;
250 my $im = PublicInbox::V2Writable->new($ibx);
251 eval { $im->index_sync() };
252 is($@, '', 'no error from reindexing without msgmap');
253 is_deeply(\@warn, [], 'no warnings');
255 my ($min, $max) = $ibx->mm->minmax;
256 is($min, 1, 'min as expected');
257 is($max, 4, 'max as expected');
258 is($ibx->mm->num_highwater, 4, 'num_highwater as expected');
259 is_deeply($ibx->mm->msg_range(\$min, $max),
261 [1, '1@example.com' ],
262 [2, '2@example.com' ],
263 [3, '3@example.com' ],
264 [4, '4@example.com' ],
265 ], 'msgmap as expected' );
269 local $SIG{__WARN__} = sub { push @warn, @_ };
270 my %config = %$ibx_config;
271 my $ibx = PublicInbox::Inbox->new(\%config);
272 # mark2 A delete separated from an add in the same index_sync
273 $ibx->{ref_head} = $mark2;
274 my $im = PublicInbox::V2Writable->new($ibx);
275 eval { $im->index_sync() };
276 is($@, '', 'no error from reindexing without msgmap');
277 is_deeply(\@warn, [], 'no warnings');
279 my ($min, $max) = $ibx->mm->minmax;
280 is($min, 1, 'min as expected');
281 is($max, 3, 'max as expected');
282 is($ibx->mm->num_highwater, 4, 'num_highwater as expected');
283 is_deeply($ibx->mm->msg_range(\$min, $max),
285 [1, '1@example.com' ],
286 [2, '2@example.com' ],
287 [3, '3@example.com' ],
288 ], 'msgmap as expected' );
292 local $SIG{__WARN__} = sub { push @warn, @_ };
293 my %config = %$ibx_config;
294 my $ibx = PublicInbox::Inbox->new(\%config);
295 # mark3 adds following the delete at mark2
296 $ibx->{ref_head} = $mark3;
297 my $im = PublicInbox::V2Writable->new($ibx);
298 eval { $im->index_sync() };
299 is($@, '', 'no error from reindexing without msgmap');
300 is_deeply(\@warn, [], 'no warnings');
302 my ($min, $max) = $ibx->mm->minmax;
303 is($min, 1, 'min as expected');
304 is($max, 10, 'max as expected');
305 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
306 is_deeply($ibx->mm->msg_range(\$min, $max),
308 [1, '1@example.com' ],
309 [2, '2@example.com' ],
310 [3, '3@example.com' ],
311 [5, '5@example.com' ],
312 [6, '6@example.com' ],
313 [7, '7@example.com' ],
314 [8, '8@example.com' ],
315 [9, '9@example.com' ],
316 [10, '10@example.com' ],
317 ], 'msgmap as expected' );
321 local $SIG{__WARN__} = sub { push @warn, @_ };
322 my %config = %$ibx_config;
323 my $ibx = PublicInbox::Inbox->new(\%config);
324 # mark4 A delete of an older message
325 $ibx->{ref_head} = $mark4;
326 my $im = PublicInbox::V2Writable->new($ibx);
327 eval { $im->index_sync() };
328 is($@, '', 'no error from reindexing without msgmap');
329 is_deeply(\@warn, [], 'no warnings');
331 my ($min, $max) = $ibx->mm->minmax;
332 is($min, 1, 'min as expected');
333 is($max, 10, 'max as expected');
334 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
335 is_deeply($ibx->mm->msg_range(\$min, $max),
337 [1, '1@example.com' ],
338 [2, '2@example.com' ],
339 [3, '3@example.com' ],
340 [6, '6@example.com' ],
341 [7, '7@example.com' ],
342 [8, '8@example.com' ],
343 [9, '9@example.com' ],
344 [10, '10@example.com' ],
345 ], 'msgmap as expected' );
349 # Another incremental indexing test
350 ok(unlink "$inboxdir/msgmap.sqlite3", 'remove msgmap');
352 ok(!-d $xap, 'Xapian directories removed again');
355 local $SIG{__WARN__} = sub { push @warn, @_ };
356 my %config = %$ibx_config;
357 my $ibx = PublicInbox::Inbox->new(\%config);
358 # mark2 an add and it's delete in the same index_sync
359 $ibx->{ref_head} = $mark2;
360 my $im = PublicInbox::V2Writable->new($ibx);
361 eval { $im->index_sync() };
362 is($@, '', 'no error from reindexing without msgmap');
363 is_deeply(\@warn, [], 'no warnings');
365 my ($min, $max) = $ibx->mm->minmax;
366 is($min, 1, 'min as expected');
367 is($max, 3, 'max as expected');
368 is($ibx->mm->num_highwater, 4, 'num_highwater as expected');
369 is_deeply($ibx->mm->msg_range(\$min, $max),
371 [1, '1@example.com' ],
372 [2, '2@example.com' ],
373 [3, '3@example.com' ],
374 ], 'msgmap as expected' );
378 local $SIG{__WARN__} = sub { push @warn, @_ };
379 my %config = %$ibx_config;
380 my $ibx = PublicInbox::Inbox->new(\%config);
381 # mark3 adds following the delete at mark2
382 $ibx->{ref_head} = $mark3;
383 my $im = PublicInbox::V2Writable->new($ibx);
384 eval { $im->index_sync() };
385 is($@, '', 'no error from reindexing without msgmap');
386 is_deeply(\@warn, [], 'no warnings');
388 my ($min, $max) = $ibx->mm->minmax;
389 is($min, 1, 'min as expected');
390 is($max, 10, 'max as expected');
391 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
392 is_deeply($ibx->mm->msg_range(\$min, $max),
394 [1, '1@example.com' ],
395 [2, '2@example.com' ],
396 [3, '3@example.com' ],
397 [5, '5@example.com' ],
398 [6, '6@example.com' ],
399 [7, '7@example.com' ],
400 [8, '8@example.com' ],
401 [9, '9@example.com' ],
402 [10, '10@example.com' ],
403 ], 'msgmap as expected' );
407 local $SIG{__WARN__} = sub { push @warn, @_ };
408 my %config = %$ibx_config;
409 my $ibx = PublicInbox::Inbox->new(\%config);
410 # mark4 A delete of an older message
411 $ibx->{ref_head} = $mark4;
412 my $im = PublicInbox::V2Writable->new($ibx);
413 eval { $im->index_sync() };
414 is($@, '', 'no error from reindexing without msgmap');
415 is_deeply(\@warn, [], 'no warnings');
417 my ($min, $max) = $ibx->mm->minmax;
418 is($min, 1, 'min as expected');
419 is($max, 10, 'max as expected');
420 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
421 is_deeply($ibx->mm->msg_range(\$min, $max),
423 [1, '1@example.com' ],
424 [2, '2@example.com' ],
425 [3, '3@example.com' ],
426 [6, '6@example.com' ],
427 [7, '7@example.com' ],
428 [8, '8@example.com' ],
429 [9, '9@example.com' ],
430 [10, '10@example.com' ],
431 ], 'msgmap as expected' );
434 # A real example from linux-renesas-soc on lore where a 3-headed monster
435 # of a message has 3 sets of common headers. Another normal message
436 # previously existed with a single Message-ID that conflicts with one
437 # of the Message-IDs in the 3-headed monster.
440 local $SIG{__WARN__} = sub { push @warn, @_ };
441 my %config = %$ibx_config;
442 $config{indexlevel} = 'medium';
443 my $ibx = PublicInbox::Inbox->new(\%config);
444 my $im = PublicInbox::V2Writable->new($ibx);
445 my $m3 = PublicInbox::MIME->new(<<'EOF');
446 Date: Tue, 24 May 2016 14:34:22 -0700 (PDT)
447 Message-Id: <20160524.143422.552507610109476444.d@example.com>
450 Subject: Re: [PATCH v2 2/2] uno
451 From: <f@example.com>
452 In-Reply-To: <1463825855-7363-2-git-send-email-y@example.com>
453 References: <1463825855-7363-1-git-send-email-y@example.com>
454 <1463825855-7363-2-git-send-email-y@example.com>
455 Date: Wed, 25 May 2016 10:01:51 +0900
459 Subject: Re: [PATCH] dos
460 Message-ID: <20160525010150.GD7292@example.com>
461 References: <1463498133-23918-1-git-send-email-g+r@example.com>
462 In-Reply-To: <1463498133-23918-1-git-send-email-g+r@example.com>
466 Subject: [PATCH 12/13] tres
467 Date: Wed, 01 Jun 2016 01:32:35 +0300
468 Message-ID: <1923946.Jvi0TDUXFC@wasted.example.com>
469 In-Reply-To: <13205049.n7pM8utpHF@wasted.example.com>
470 References: <13205049.n7pM8utpHF@wasted.example.com>
472 Somehow we got a message with 3 sets of headers into one
473 message, could've been something broken on the archiver side.
476 my $m1 = PublicInbox::MIME->new(<<'EOF');
479 Subject: [PATCH 12/13]
480 Date: Wed, 01 Jun 2016 01:32:35 +0300
481 Message-ID: <1923946.Jvi0TDUXFC@wasted.example.com>
482 In-Reply-To: <13205049.n7pM8utpHF@wasted.example.com>
483 References: <13205049.n7pM8utpHF@wasted.example.com>
485 This is probably one of the original messages
492 eval { $im->index_sync() };
493 is($@, '', 'no error from initial indexing');
494 is_deeply(\@warn, [], 'no warnings from initial index');
495 eval { $im->index_sync({reindex=>1}) };
496 is($@, '', 'no error from reindexing after reused Message-ID (x3)');
497 is_deeply(\@warn, [], 'no warnings on reindex');
500 for my $s (qw(uno dos tres)) {
501 my $msgs = $ibx->search->query("s:$s");
502 is(scalar(@$msgs), 1, "only one result for `$s'");
503 $uniq{$msgs->[0]->{num}}++;
505 is_deeply([values %uniq], [3], 'search on different subjects');