1 # Copyright (C) 2018 all contributors <meta@public-inbox.org>
2 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
7 use PublicInbox::ContentId qw(content_digest);
8 use File::Temp qw/tempdir/;
9 use File::Path qw(remove_tree);
11 foreach my $mod (qw(DBD::SQLite Search::Xapian)) {
13 plan skip_all => "$mod missing for v2reindex.t" if $@;
15 use_ok 'PublicInbox::V2Writable';
16 my $mainrepo = tempdir('pi-v2reindex-XXXXXX', TMPDIR => 1, CLEANUP => 1);
18 mainrepo => $mainrepo,
19 name => 'test-v2writable',
21 -primary_address => 'test@example.com',
24 my $mime = PublicInbox::MIME->create(
26 From => 'a@example.com',
27 To => 'test@example.com',
28 Subject => 'this is a subject',
29 Date => 'Fri, 02 Oct 1993 00:00:00 +0000',
31 body => "hello world\n",
33 local $ENV{NPROC} = 2;
36 my ($mark1, $mark2, $mark3, $mark4);
38 my %config = %$ibx_config;
39 my $ibx = PublicInbox::Inbox->new(\%config);
40 my $im = PublicInbox::V2Writable->new($ibx, 1);
41 my $im0 = $im->importer();
42 foreach my $i (1..10) {
43 $mime->header_set('Message-Id', "<$i\@example.com>");
44 ok($im->add($mime), "message $i added");
46 $mark1 = $im0->get_mark($im0->{tip});
48 $mark2 = $im0->get_mark($im0->{tip});
52 if ('test remove later') {
53 $mark3 = $im0->get_mark($im0->{tip});
54 $mime->header_set('Message-Id', "<5\@example.com>");
56 $mark4 = $im0->get_mark($im0->{tip});
60 $minmax = [ $ibx->mm->minmax ];
61 ok(defined $minmax->[0] && defined $minmax->[1], 'minmax defined');
62 is_deeply($minmax, [ 1, 10 ], 'minmax as expected');
63 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
65 my ($min, $max) = @$minmax;
66 $msgmap = $ibx->mm->msg_range(\$min, $max);
68 [1, '1@example.com' ],
69 [2, '2@example.com' ],
70 [3, '3@example.com' ],
71 [6, '6@example.com' ],
72 [7, '7@example.com' ],
73 [8, '8@example.com' ],
74 [9, '9@example.com' ],
75 [10, '10@example.com' ],
76 ], 'msgmap as expected');
80 my %config = %$ibx_config;
81 my $ibx = PublicInbox::Inbox->new(\%config);
82 my $im = PublicInbox::V2Writable->new($ibx, 1);
83 eval { $im->index_sync({reindex => 1}) };
84 is($@, '', 'no error from reindexing');
88 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
89 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
91 my ($min, $max) = $ibx->mm->minmax;
92 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
95 my $xap = "$mainrepo/xap".PublicInbox::Search::SCHEMA_VERSION();
97 ok(!-d $xap, 'Xapian directories removed');
99 my %config = %$ibx_config;
100 my $ibx = PublicInbox::Inbox->new(\%config);
101 my $im = PublicInbox::V2Writable->new($ibx, 1);
102 eval { $im->index_sync({reindex => 1}) };
103 is($@, '', 'no error from reindexing');
105 ok(-d $xap, 'Xapian directories recreated');
108 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
109 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
111 my ($min, $max) = $ibx->mm->minmax;
112 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
115 ok(unlink "$mainrepo/msgmap.sqlite3", 'remove msgmap');
117 ok(!-d $xap, 'Xapian directories removed again');
120 local $SIG{__WARN__} = sub { push @warn, @_ };
121 my %config = %$ibx_config;
122 my $ibx = PublicInbox::Inbox->new(\%config);
123 my $im = PublicInbox::V2Writable->new($ibx, 1);
124 eval { $im->index_sync({reindex => 1}) };
125 is($@, '', 'no error from reindexing without msgmap');
126 is(scalar(@warn), 0, 'no warnings from reindexing');
128 ok(-d $xap, 'Xapian directories recreated');
130 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
131 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
133 my ($min, $max) = $ibx->mm->minmax;
134 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
138 ok(unlink "$mainrepo/msgmap.sqlite3", 'remove msgmap');
140 ok(!-d $xap, 'Xapian directories removed again');
143 local $SIG{__WARN__} = sub { push @warn, @_ };
144 my %config = %$ibx_config;
145 my $ibx = PublicInbox::Inbox->new(\%config);
146 my $im = PublicInbox::V2Writable->new($ibx, 1);
147 eval { $im->index_sync({reindex => 1}) };
148 is($@, '', 'no error from reindexing without msgmap');
149 is_deeply(\@warn, [], 'no warnings');
151 ok(-d $xap, 'Xapian directories recreated');
153 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
154 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
155 my $mset = $ibx->search->query('"hello world"', {mset=>1});
156 isnt($mset->size, 0, "phrase search succeeds on indexlevel=full");
157 for (<"$xap/*/*">) { $sizes{$ibx->{indexlevel}} += -s _ if -f $_ }
159 my ($min, $max) = $ibx->mm->minmax;
160 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
163 ok(unlink "$mainrepo/msgmap.sqlite3", 'remove msgmap');
165 ok(!-d $xap, 'Xapian directories removed again');
168 local $SIG{__WARN__} = sub { push @warn, @_ };
169 my %config = %$ibx_config;
170 $config{indexlevel} = 'medium';
171 my $ibx = PublicInbox::Inbox->new(\%config);
172 my $im = PublicInbox::V2Writable->new($ibx);
173 eval { $im->index_sync({reindex => 1}) };
174 is($@, '', 'no error from reindexing without msgmap');
175 is_deeply(\@warn, [], 'no warnings');
177 ok(-d $xap, 'Xapian directories recreated');
179 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
180 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
183 # not sure why, but Xapian seems to fallback to terms and
184 # phrase searches still work
185 delete $ibx->{search};
186 my $mset = $ibx->search->query('"hello world"', {mset=>1});
187 is($mset->size, 0, 'phrase search does not work on medium');
190 my $mset = $ibx->search->query('hello world', {mset=>1});
191 isnt($mset->size, 0, "normal search works on indexlevel=medium");
192 for (<"$xap/*/*">) { $sizes{$ibx->{indexlevel}} += -s _ if -f $_ }
193 ok($sizes{full} > $sizes{medium}, 'medium is smaller than full');
196 my ($min, $max) = $ibx->mm->minmax;
197 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
200 ok(unlink "$mainrepo/msgmap.sqlite3", 'remove msgmap');
202 ok(!-d $xap, 'Xapian directories removed again');
205 local $SIG{__WARN__} = sub { push @warn, @_ };
206 my %config = %$ibx_config;
207 $config{indexlevel} = 'basic';
208 my $ibx = PublicInbox::Inbox->new(\%config);
209 my $im = PublicInbox::V2Writable->new($ibx);
210 eval { $im->index_sync({reindex => 1}) };
211 is($@, '', 'no error from reindexing without msgmap');
212 is_deeply(\@warn, [], 'no warnings');
214 ok(-d $xap, 'Xapian directories recreated');
216 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
217 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
218 my $mset = $ibx->search->query('hello', {mset=>1});
219 is($mset->size, 0, "search fails on indexlevel='basic'");
220 for (<"$xap/*/*">) { $sizes{$ibx->{indexlevel}} += -s _ if -f $_ }
221 ok($sizes{medium} > $sizes{basic}, 'basic is smaller than medium');
223 my ($min, $max) = $ibx->mm->minmax;
224 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
228 # An incremental indexing test
229 ok(unlink "$mainrepo/msgmap.sqlite3", 'remove msgmap');
231 ok(!-d $xap, 'Xapian directories removed again');
234 local $SIG{__WARN__} = sub { push @warn, @_ };
235 my %config = %$ibx_config;
236 my $ibx = PublicInbox::Inbox->new(\%config);
237 # mark1 4 simple additions in the same index_sync
238 $ibx->{ref_head} = $mark1;
239 my $im = PublicInbox::V2Writable->new($ibx);
240 eval { $im->index_sync() };
241 is($@, '', 'no error from reindexing without msgmap');
242 is_deeply(\@warn, [], 'no warnings');
244 my ($min, $max) = $ibx->mm->minmax;
245 is($min, 1, 'min as expected');
246 is($max, 4, 'max as expected');
247 is($ibx->mm->num_highwater, 4, 'num_highwater as expected');
248 is_deeply($ibx->mm->msg_range(\$min, $max),
250 [1, '1@example.com' ],
251 [2, '2@example.com' ],
252 [3, '3@example.com' ],
253 [4, '4@example.com' ],
254 ], 'msgmap as expected' );
258 local $SIG{__WARN__} = sub { push @warn, @_ };
259 my %config = %$ibx_config;
260 my $ibx = PublicInbox::Inbox->new(\%config);
261 # mark2 A delete separated from an add in the same index_sync
262 $ibx->{ref_head} = $mark2;
263 my $im = PublicInbox::V2Writable->new($ibx);
264 eval { $im->index_sync() };
265 is($@, '', 'no error from reindexing without msgmap');
266 is_deeply(\@warn, [], 'no warnings');
268 my ($min, $max) = $ibx->mm->minmax;
269 is($min, 1, 'min as expected');
270 is($max, 3, 'max as expected');
271 is($ibx->mm->num_highwater, 4, 'num_highwater as expected');
272 is_deeply($ibx->mm->msg_range(\$min, $max),
274 [1, '1@example.com' ],
275 [2, '2@example.com' ],
276 [3, '3@example.com' ],
277 ], 'msgmap as expected' );
281 local $SIG{__WARN__} = sub { push @warn, @_ };
282 my %config = %$ibx_config;
283 my $ibx = PublicInbox::Inbox->new(\%config);
284 # mark3 adds following the delete at mark2
285 $ibx->{ref_head} = $mark3;
286 my $im = PublicInbox::V2Writable->new($ibx);
287 eval { $im->index_sync() };
288 is($@, '', 'no error from reindexing without msgmap');
289 is_deeply(\@warn, [], 'no warnings');
291 my ($min, $max) = $ibx->mm->minmax;
292 is($min, 1, 'min as expected');
293 is($max, 10, 'max as expected');
294 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
295 is_deeply($ibx->mm->msg_range(\$min, $max),
297 [1, '1@example.com' ],
298 [2, '2@example.com' ],
299 [3, '3@example.com' ],
300 [5, '5@example.com' ],
301 [6, '6@example.com' ],
302 [7, '7@example.com' ],
303 [8, '8@example.com' ],
304 [9, '9@example.com' ],
305 [10, '10@example.com' ],
306 ], 'msgmap as expected' );
310 local $SIG{__WARN__} = sub { push @warn, @_ };
311 my %config = %$ibx_config;
312 my $ibx = PublicInbox::Inbox->new(\%config);
313 # mark4 A delete of an older message
314 $ibx->{ref_head} = $mark4;
315 my $im = PublicInbox::V2Writable->new($ibx);
316 eval { $im->index_sync() };
317 is($@, '', 'no error from reindexing without msgmap');
318 is_deeply(\@warn, [], 'no warnings');
320 my ($min, $max) = $ibx->mm->minmax;
321 is($min, 1, 'min as expected');
322 is($max, 10, 'max as expected');
323 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
324 is_deeply($ibx->mm->msg_range(\$min, $max),
326 [1, '1@example.com' ],
327 [2, '2@example.com' ],
328 [3, '3@example.com' ],
329 [6, '6@example.com' ],
330 [7, '7@example.com' ],
331 [8, '8@example.com' ],
332 [9, '9@example.com' ],
333 [10, '10@example.com' ],
334 ], 'msgmap as expected' );
338 # Another incremental indexing test
339 ok(unlink "$mainrepo/msgmap.sqlite3", 'remove msgmap');
341 ok(!-d $xap, 'Xapian directories removed again');
344 local $SIG{__WARN__} = sub { push @warn, @_ };
345 my %config = %$ibx_config;
346 my $ibx = PublicInbox::Inbox->new(\%config);
347 # mark2 an add and it's delete in the same index_sync
348 $ibx->{ref_head} = $mark2;
349 my $im = PublicInbox::V2Writable->new($ibx);
350 eval { $im->index_sync() };
351 is($@, '', 'no error from reindexing without msgmap');
352 is_deeply(\@warn, [], 'no warnings');
354 my ($min, $max) = $ibx->mm->minmax;
355 is($min, 1, 'min as expected');
356 is($max, 3, 'max as expected');
357 is($ibx->mm->num_highwater, 4, 'num_highwater as expected');
358 is_deeply($ibx->mm->msg_range(\$min, $max),
360 [1, '1@example.com' ],
361 [2, '2@example.com' ],
362 [3, '3@example.com' ],
363 ], 'msgmap as expected' );
367 local $SIG{__WARN__} = sub { push @warn, @_ };
368 my %config = %$ibx_config;
369 my $ibx = PublicInbox::Inbox->new(\%config);
370 # mark3 adds following the delete at mark2
371 $ibx->{ref_head} = $mark3;
372 my $im = PublicInbox::V2Writable->new($ibx);
373 eval { $im->index_sync() };
374 is($@, '', 'no error from reindexing without msgmap');
375 is_deeply(\@warn, [], 'no warnings');
377 my ($min, $max) = $ibx->mm->minmax;
378 is($min, 1, 'min as expected');
379 is($max, 10, 'max as expected');
380 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
381 is_deeply($ibx->mm->msg_range(\$min, $max),
383 [1, '1@example.com' ],
384 [2, '2@example.com' ],
385 [3, '3@example.com' ],
386 [5, '5@example.com' ],
387 [6, '6@example.com' ],
388 [7, '7@example.com' ],
389 [8, '8@example.com' ],
390 [9, '9@example.com' ],
391 [10, '10@example.com' ],
392 ], 'msgmap as expected' );
396 local $SIG{__WARN__} = sub { push @warn, @_ };
397 my %config = %$ibx_config;
398 my $ibx = PublicInbox::Inbox->new(\%config);
399 # mark4 A delete of an older message
400 $ibx->{ref_head} = $mark4;
401 my $im = PublicInbox::V2Writable->new($ibx);
402 eval { $im->index_sync() };
403 is($@, '', 'no error from reindexing without msgmap');
404 is_deeply(\@warn, [], 'no warnings');
406 my ($min, $max) = $ibx->mm->minmax;
407 is($min, 1, 'min as expected');
408 is($max, 10, 'max as expected');
409 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
410 is_deeply($ibx->mm->msg_range(\$min, $max),
412 [1, '1@example.com' ],
413 [2, '2@example.com' ],
414 [3, '3@example.com' ],
415 [6, '6@example.com' ],
416 [7, '7@example.com' ],
417 [8, '8@example.com' ],
418 [9, '9@example.com' ],
419 [10, '10@example.com' ],
420 ], 'msgmap as expected' );