1 # Copyright (C) 2018 all contributors <meta@public-inbox.org>
2 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
7 use PublicInbox::ContentId qw(content_digest);
8 use File::Temp qw/tempdir/;
9 use File::Path qw(remove_tree);
10 require './t/common.perl';
13 foreach my $mod (qw(DBD::SQLite Search::Xapian)) {
15 plan skip_all => "$mod missing for v1reindex.t" if $@;
17 use_ok 'PublicInbox::SearchIdx';
18 use_ok 'PublicInbox::Import';
19 my $mainrepo = tempdir('pi-v1reindex-XXXXXX', TMPDIR => 1, CLEANUP => 1);
20 is(system(qw(git init -q --bare), $mainrepo), 0);
22 mainrepo => $mainrepo,
23 name => 'test-v1reindex',
24 -primary_address => 'test@example.com',
27 my $mime = PublicInbox::MIME->create(
29 From => 'a@example.com',
30 To => 'test@example.com',
31 Subject => 'this is a subject',
32 Date => 'Fri, 02 Oct 1993 00:00:00 +0000',
34 body => "hello world\n",
38 my ($mark1, $mark2, $mark3, $mark4);
40 my %config = %$ibx_config;
41 my $ibx = PublicInbox::Inbox->new(\%config);
42 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
43 foreach my $i (1..10) {
44 $mime->header_set('Message-Id', "<$i\@example.com>");
45 ok($im->add($mime), "message $i added");
47 $mark1 = $im->get_mark($im->{tip});
49 $mark2 = $im->get_mark($im->{tip});
53 if ('test remove later') {
54 $mark3 = $im->get_mark($im->{tip});
55 $mime->header_set('Message-Id', "<5\@example.com>");
57 $mark4 = $im->get_mark($im->{tip});
61 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
62 eval { $rw->index_sync() };
63 is($@, '', 'no error from indexing');
65 $minmax = [ $ibx->mm->minmax ];
66 ok(defined $minmax->[0] && defined $minmax->[1], 'minmax defined');
67 is_deeply($minmax, [ 1, 10 ], 'minmax as expected');
68 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
70 my ($min, $max) = @$minmax;
71 $msgmap = $ibx->mm->msg_range(\$min, $max);
73 [1, '1@example.com' ],
74 [2, '2@example.com' ],
75 [3, '3@example.com' ],
76 [6, '6@example.com' ],
77 [7, '7@example.com' ],
78 [8, '8@example.com' ],
79 [9, '9@example.com' ],
80 [10, '10@example.com' ],
81 ], 'msgmap as expected');
85 my %config = %$ibx_config;
86 my $ibx = PublicInbox::Inbox->new(\%config);
87 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
88 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
89 eval { $rw->index_sync({reindex => 1}) };
90 is($@, '', 'no error from reindexing');
93 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
95 my ($min, $max) = $ibx->mm->minmax;
96 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
99 my $xap = "$mainrepo/public-inbox/xapian".PublicInbox::Search::SCHEMA_VERSION();
101 ok(!-d $xap, 'Xapian directories removed');
103 my %config = %$ibx_config;
104 my $ibx = PublicInbox::Inbox->new(\%config);
105 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
106 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
108 eval { $rw->index_sync({reindex => 1}) };
109 is($@, '', 'no error from reindexing');
111 ok(-d $xap, 'Xapian directories recreated');
114 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
115 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
117 my ($min, $max) = $ibx->mm->minmax;
118 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
121 ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
123 ok(!-d $xap, 'Xapian directories removed again');
126 local $SIG{__WARN__} = sub { push @warn, @_ };
127 my %config = %$ibx_config;
128 my $ibx = PublicInbox::Inbox->new(\%config);
129 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
130 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
131 eval { $rw->index_sync({reindex => 1}) };
132 is($@, '', 'no error from reindexing without msgmap');
133 is(scalar(@warn), 0, 'no warnings from reindexing');
135 ok(-d $xap, 'Xapian directories recreated');
137 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
138 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
140 my ($min, $max) = $ibx->mm->minmax;
141 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
144 ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
146 ok(!-d $xap, 'Xapian directories removed again');
149 local $SIG{__WARN__} = sub { push @warn, @_ };
150 my %config = %$ibx_config;
151 my $ibx = PublicInbox::Inbox->new(\%config);
152 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
153 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
154 eval { $rw->index_sync({reindex => 1}) };
155 is($@, '', 'no error from reindexing without msgmap');
156 is_deeply(\@warn, [], 'no warnings');
158 ok(-d $xap, 'Xapian directories recreated');
160 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
161 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
163 my ($min, $max) = @$minmax;
164 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
167 ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
169 ok(!-d $xap, 'Xapian directories removed again');
172 local $SIG{__WARN__} = sub { push @warn, @_ };
173 my %config = %$ibx_config;
174 $config{indexlevel} = 'medium';
175 my $ibx = PublicInbox::Inbox->new(\%config);
176 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
177 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
178 eval { $rw->index_sync({reindex => 1}) };
179 is($@, '', 'no error from reindexing without msgmap');
180 is_deeply(\@warn, [], 'no warnings');
182 ok(-d $xap, 'Xapian directories recreated');
184 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
185 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
186 my $mset = $ibx->search->query('hello world', {mset=>1});
187 isnt($mset->size, 0, 'got Xapian search results');
189 my ($min, $max) = $ibx->mm->minmax;
190 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
193 ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
195 ok(!-d $xap, 'Xapian directories removed again');
198 local $SIG{__WARN__} = sub { push @warn, @_ };
199 my %config = %$ibx_config;
200 $config{indexlevel} = 'basic';
201 my $ibx = PublicInbox::Inbox->new(\%config);
202 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
203 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
204 eval { $rw->index_sync({reindex => 1}) };
205 is($@, '', 'no error from reindexing without msgmap');
206 is_deeply(\@warn, [], 'no warnings');
208 ok(-d $xap, 'Xapian directories recreated');
210 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
211 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
212 my $mset = $ibx->search->reopen->query('hello world', {mset=>1});
213 is($mset->size, 0, "no Xapian search results");
215 my ($min, $max) = $ibx->mm->minmax;
216 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
219 # upgrade existing basic to medium
220 # note: changing indexlevels is not yet supported in v2,
221 # and may not be without more effort
225 local $SIG{__WARN__} = sub { push @warn, @_ };
226 my %config = %$ibx_config;
227 $config{indexleve} = 'medium';
228 my $ibx = PublicInbox::Inbox->new(\%config);
229 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
230 eval { $rw->index_sync };
231 is($@, '', 'no error from indexing');
232 is_deeply(\@warn, [], 'no warnings');
233 my $mset = $ibx->search->reopen->query('hello world', {mset=>1});
234 isnt($mset->size, 0, 'search OK after basic -> medium');
236 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
238 my ($min, $max) = $ibx->mm->minmax;
239 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
242 # An incremental indexing test
243 ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
245 ok(!-d $xap, 'Xapian directories removed again');
248 local $SIG{__WARN__} = sub { push @warn, @_ };
249 my %config = %$ibx_config;
250 my $ibx = PublicInbox::Inbox->new(\%config);
251 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
252 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
253 # mark1 4 simple additions in the same index_sync
254 eval { $rw->index_sync({ref => $mark1}) };
255 is($@, '', 'no error from reindexing without msgmap');
256 is_deeply(\@warn, [], 'no warnings');
258 my ($min, $max) = $ibx->mm->minmax;
259 is($min, 1, 'min as expected');
260 is($max, 4, 'max as expected');
261 is($ibx->mm->num_highwater, 4, 'num_highwater as expected');
262 is_deeply($ibx->mm->msg_range(\$min, $max),
264 [1, '1@example.com' ],
265 [2, '2@example.com' ],
266 [3, '3@example.com' ],
267 [4, '4@example.com' ],
268 ], 'msgmap as expected' );
272 local $SIG{__WARN__} = sub { push @warn, @_ };
273 my %config = %$ibx_config;
274 my $ibx = PublicInbox::Inbox->new(\%config);
275 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
276 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
277 # mark2 A delete separated form and add in the same index_sync
278 eval { $rw->index_sync({ref => $mark2}) };
279 is($@, '', 'no error from reindexing without msgmap');
280 is_deeply(\@warn, [], 'no warnings');
282 my ($min, $max) = $ibx->mm->minmax;
283 is($min, 1, 'min as expected');
284 is($max, 3, 'max as expected');
285 is($ibx->mm->num_highwater, 4, 'num_highwater as expected');
286 is_deeply($ibx->mm->msg_range(\$min, $max),
288 [1, '1@example.com' ],
289 [2, '2@example.com' ],
290 [3, '3@example.com' ],
291 ], 'msgmap as expected' );
295 local $SIG{__WARN__} = sub { push @warn, @_ };
296 my %config = %$ibx_config;
297 my $ibx = PublicInbox::Inbox->new(\%config);
298 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
299 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
300 # mark3 adds following the delete at mark2
301 eval { $rw->index_sync({ref => $mark3}) };
302 is($@, '', 'no error from reindexing without msgmap');
303 is_deeply(\@warn, [], 'no warnings');
305 my ($min, $max) = $ibx->mm->minmax;
306 is($min, 1, 'min as expected');
307 is($max, 10, 'max as expected');
308 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
309 is_deeply($ibx->mm->msg_range(\$min, $max),
311 [1, '1@example.com' ],
312 [2, '2@example.com' ],
313 [3, '3@example.com' ],
314 [5, '5@example.com' ],
315 [6, '6@example.com' ],
316 [7, '7@example.com' ],
317 [8, '8@example.com' ],
318 [9, '9@example.com' ],
319 [10, '10@example.com' ],
320 ], 'msgmap as expected' );
324 local $SIG{__WARN__} = sub { push @warn, @_ };
325 my %config = %$ibx_config;
326 my $ibx = PublicInbox::Inbox->new(\%config);
327 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
328 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
329 # mark4 A delete of an older message
330 eval { $rw->index_sync({ref => $mark4}) };
331 is($@, '', 'no error from reindexing without msgmap');
332 is_deeply(\@warn, [], 'no warnings');
334 my ($min, $max) = $ibx->mm->minmax;
335 is($min, 1, 'min as expected');
336 is($max, 10, 'max as expected');
337 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
338 is_deeply($ibx->mm->msg_range(\$min, $max),
340 [1, '1@example.com' ],
341 [2, '2@example.com' ],
342 [3, '3@example.com' ],
343 [6, '6@example.com' ],
344 [7, '7@example.com' ],
345 [8, '8@example.com' ],
346 [9, '9@example.com' ],
347 [10, '10@example.com' ],
348 ], 'msgmap as expected' );
352 # Another incremental indexing test
353 ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
355 ok(!-d $xap, 'Xapian directories removed again');
358 local $SIG{__WARN__} = sub { push @warn, @_ };
359 my %config = %$ibx_config;
360 my $ibx = PublicInbox::Inbox->new(\%config);
361 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
362 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
363 # mark2 an add and it's delete in the same index_sync
364 eval { $rw->index_sync({ref => $mark2}) };
365 is($@, '', 'no error from reindexing without msgmap');
366 is_deeply(\@warn, [], 'no warnings');
368 my ($min, $max) = $ibx->mm->minmax;
369 is($min, 1, 'min as expected');
370 is($max, 3, 'max as expected');
371 is($ibx->mm->num_highwater, 4, 'num_highwater as expected');
372 is_deeply($ibx->mm->msg_range(\$min, $max),
374 [1, '1@example.com' ],
375 [2, '2@example.com' ],
376 [3, '3@example.com' ],
377 ], 'msgmap as expected' );
381 local $SIG{__WARN__} = sub { push @warn, @_ };
382 my %config = %$ibx_config;
383 my $ibx = PublicInbox::Inbox->new(\%config);
384 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
385 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
386 # mark3 adds following the delete at mark2
387 eval { $rw->index_sync({ref => $mark3}) };
388 is($@, '', 'no error from reindexing without msgmap');
389 is_deeply(\@warn, [], 'no warnings');
391 my ($min, $max) = $ibx->mm->minmax;
392 is($min, 1, 'min as expected');
393 is($max, 10, 'max as expected');
394 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
395 is_deeply($ibx->mm->msg_range(\$min, $max),
397 [1, '1@example.com' ],
398 [2, '2@example.com' ],
399 [3, '3@example.com' ],
400 [5, '5@example.com' ],
401 [6, '6@example.com' ],
402 [7, '7@example.com' ],
403 [8, '8@example.com' ],
404 [9, '9@example.com' ],
405 [10, '10@example.com' ],
406 ], 'msgmap as expected' );
410 local $SIG{__WARN__} = sub { push @warn, @_ };
411 my %config = %$ibx_config;
412 my $ibx = PublicInbox::Inbox->new(\%config);
413 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
414 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
415 # mark4 A delete of an older message
416 eval { $rw->index_sync({ref => $mark4}) };
417 is($@, '', 'no error from reindexing without msgmap');
418 is_deeply(\@warn, [], 'no warnings');
420 my ($min, $max) = $ibx->mm->minmax;
421 is($min, 1, 'min as expected');
422 is($max, 10, 'max as expected');
423 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
424 is_deeply($ibx->mm->msg_range(\$min, $max),
426 [1, '1@example.com' ],
427 [2, '2@example.com' ],
428 [3, '3@example.com' ],
429 [6, '6@example.com' ],
430 [7, '7@example.com' ],
431 [8, '8@example.com' ],
432 [9, '9@example.com' ],
433 [10, '10@example.com' ],
434 ], 'msgmap as expected' );