1 # Copyright (C) 2018 all contributors <meta@public-inbox.org>
2 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
7 use PublicInbox::ContentId qw(content_digest);
8 use File::Temp qw/tempdir/;
9 use File::Path qw(remove_tree);
11 foreach my $mod (qw(DBD::SQLite Search::Xapian)) {
13 plan skip_all => "$mod missing for v1reindex.t" if $@;
15 use_ok 'PublicInbox::SearchIdx';
16 use_ok 'PublicInbox::Import';
17 my $mainrepo = tempdir('pi-v1reindex-XXXXXX', TMPDIR => 1, CLEANUP => 1);
18 is(system(qw(git init -q --bare), $mainrepo), 0);
20 mainrepo => $mainrepo,
21 name => 'test-v1reindex',
22 -primary_address => 'test@example.com',
25 my $mime = PublicInbox::MIME->create(
27 From => 'a@example.com',
28 To => 'test@example.com',
29 Subject => 'this is a subject',
30 Date => 'Fri, 02 Oct 1993 00:00:00 +0000',
32 body => "hello world\n",
36 my ($mark1, $mark2, $mark3, $mark4);
38 my %config = %$ibx_config;
39 my $ibx = PublicInbox::Inbox->new(\%config);
40 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
41 foreach my $i (1..10) {
42 $mime->header_set('Message-Id', "<$i\@example.com>");
43 ok($im->add($mime), "message $i added");
45 $mark1 = $im->get_mark($im->{tip});
47 $mark2 = $im->get_mark($im->{tip});
51 if ('test remove later') {
52 $mark3 = $im->get_mark($im->{tip});
53 $mime->header_set('Message-Id', "<5\@example.com>");
55 $mark4 = $im->get_mark($im->{tip});
59 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
60 eval { $rw->index_sync() };
61 is($@, '', 'no error from indexing');
63 $minmax = [ $ibx->mm->minmax ];
64 ok(defined $minmax->[0] && defined $minmax->[1], 'minmax defined');
65 is_deeply($minmax, [ 1, 10 ], 'minmax as expected');
66 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
68 my ($min, $max) = @$minmax;
69 $msgmap = $ibx->mm->msg_range(\$min, $max);
71 [1, '1@example.com' ],
72 [2, '2@example.com' ],
73 [3, '3@example.com' ],
74 [6, '6@example.com' ],
75 [7, '7@example.com' ],
76 [8, '8@example.com' ],
77 [9, '9@example.com' ],
78 [10, '10@example.com' ],
79 ], 'msgmap as expected');
83 my %config = %$ibx_config;
84 my $ibx = PublicInbox::Inbox->new(\%config);
85 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
86 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
87 eval { $rw->index_sync({reindex => 1}) };
88 is($@, '', 'no error from reindexing');
91 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
93 my ($min, $max) = $ibx->mm->minmax;
94 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
97 my $xap = "$mainrepo/public-inbox/xapian".PublicInbox::Search::SCHEMA_VERSION();
99 ok(!-d $xap, 'Xapian directories removed');
101 my %config = %$ibx_config;
102 my $ibx = PublicInbox::Inbox->new(\%config);
103 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
104 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
106 eval { $rw->index_sync({reindex => 1}) };
107 is($@, '', 'no error from reindexing');
109 ok(-d $xap, 'Xapian directories recreated');
112 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
113 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
115 my ($min, $max) = $ibx->mm->minmax;
116 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
119 ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
121 ok(!-d $xap, 'Xapian directories removed again');
124 local $SIG{__WARN__} = sub { push @warn, @_ };
125 my %config = %$ibx_config;
126 my $ibx = PublicInbox::Inbox->new(\%config);
127 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
128 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
129 eval { $rw->index_sync({reindex => 1}) };
130 is($@, '', 'no error from reindexing without msgmap');
131 is(scalar(@warn), 0, 'no warnings from reindexing');
133 ok(-d $xap, 'Xapian directories recreated');
135 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
136 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
138 my ($min, $max) = $ibx->mm->minmax;
139 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
142 ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
144 ok(!-d $xap, 'Xapian directories removed again');
147 local $SIG{__WARN__} = sub { push @warn, @_ };
148 my %config = %$ibx_config;
149 my $ibx = PublicInbox::Inbox->new(\%config);
150 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
151 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
152 eval { $rw->index_sync({reindex => 1}) };
153 is($@, '', 'no error from reindexing without msgmap');
154 is_deeply(\@warn, [], 'no warnings');
156 ok(-d $xap, 'Xapian directories recreated');
158 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
159 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
161 my ($min, $max) = @$minmax;
162 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
165 ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
167 ok(!-d $xap, 'Xapian directories removed again');
170 local $SIG{__WARN__} = sub { push @warn, @_ };
171 my %config = %$ibx_config;
172 $config{indexlevel} = 'medium';
173 my $ibx = PublicInbox::Inbox->new(\%config);
174 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
175 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
176 eval { $rw->index_sync({reindex => 1}) };
177 is($@, '', 'no error from reindexing without msgmap');
178 is_deeply(\@warn, [], 'no warnings');
180 ok(-d $xap, 'Xapian directories recreated');
182 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
183 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
184 my $mset = $ibx->search->query('hello world', {mset=>1});
185 isnt($mset->size, 0, 'got Xapian search results');
187 my ($min, $max) = $ibx->mm->minmax;
188 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
191 ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
193 ok(!-d $xap, 'Xapian directories removed again');
196 local $SIG{__WARN__} = sub { push @warn, @_ };
197 my %config = %$ibx_config;
198 $config{indexlevel} = 'basic';
199 my $ibx = PublicInbox::Inbox->new(\%config);
200 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
201 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
202 eval { $rw->index_sync({reindex => 1}) };
203 is($@, '', 'no error from reindexing without msgmap');
204 is_deeply(\@warn, [], 'no warnings');
206 ok(-d $xap, 'Xapian directories recreated');
208 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
209 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
210 my $mset = $ibx->search->reopen->query('hello world', {mset=>1});
211 is($mset->size, 0, "no Xapian search results");
213 my ($min, $max) = $ibx->mm->minmax;
214 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
217 # upgrade existing basic to medium
218 # note: changing indexlevels is not yet supported in v2,
219 # and may not be without more effort
223 local $SIG{__WARN__} = sub { push @warn, @_ };
224 my %config = %$ibx_config;
225 $config{indexleve} = 'medium';
226 my $ibx = PublicInbox::Inbox->new(\%config);
227 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
228 eval { $rw->index_sync };
229 is($@, '', 'no error from indexing');
230 is_deeply(\@warn, [], 'no warnings');
231 my $mset = $ibx->search->reopen->query('hello world', {mset=>1});
232 isnt($mset->size, 0, 'search OK after basic -> medium');
234 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
236 my ($min, $max) = $ibx->mm->minmax;
237 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
240 # An incremental indexing test
241 ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
243 ok(!-d $xap, 'Xapian directories removed again');
246 local $SIG{__WARN__} = sub { push @warn, @_ };
247 my %config = %$ibx_config;
248 my $ibx = PublicInbox::Inbox->new(\%config);
249 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
250 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
251 # mark1 4 simple additions in the same index_sync
252 eval { $rw->index_sync({ref => $mark1}) };
253 is($@, '', 'no error from reindexing without msgmap');
254 is_deeply(\@warn, [], 'no warnings');
256 my ($min, $max) = $ibx->mm->minmax;
257 is($min, 1, 'min as expected');
258 is($max, 4, 'max as expected');
259 is($ibx->mm->num_highwater, 4, 'num_highwater as expected');
260 is_deeply($ibx->mm->msg_range(\$min, $max),
262 [1, '1@example.com' ],
263 [2, '2@example.com' ],
264 [3, '3@example.com' ],
265 [4, '4@example.com' ],
266 ], 'msgmap as expected' );
270 local $SIG{__WARN__} = sub { push @warn, @_ };
271 my %config = %$ibx_config;
272 my $ibx = PublicInbox::Inbox->new(\%config);
273 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
274 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
275 # mark2 A delete separated form and add in the same index_sync
276 eval { $rw->index_sync({ref => $mark2}) };
277 is($@, '', 'no error from reindexing without msgmap');
278 is_deeply(\@warn, [], 'no warnings');
280 my ($min, $max) = $ibx->mm->minmax;
281 is($min, 1, 'min as expected');
282 is($max, 3, 'max as expected');
283 is($ibx->mm->num_highwater, 4, 'num_highwater as expected');
284 is_deeply($ibx->mm->msg_range(\$min, $max),
286 [1, '1@example.com' ],
287 [2, '2@example.com' ],
288 [3, '3@example.com' ],
289 ], 'msgmap as expected' );
293 local $SIG{__WARN__} = sub { push @warn, @_ };
294 my %config = %$ibx_config;
295 my $ibx = PublicInbox::Inbox->new(\%config);
296 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
297 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
298 # mark3 adds following the delete at mark2
299 eval { $rw->index_sync({ref => $mark3}) };
300 is($@, '', 'no error from reindexing without msgmap');
301 is_deeply(\@warn, [], 'no warnings');
303 my ($min, $max) = $ibx->mm->minmax;
304 is($min, 1, 'min as expected');
305 is($max, 10, 'max as expected');
306 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
307 is_deeply($ibx->mm->msg_range(\$min, $max),
309 [1, '1@example.com' ],
310 [2, '2@example.com' ],
311 [3, '3@example.com' ],
312 [5, '5@example.com' ],
313 [6, '6@example.com' ],
314 [7, '7@example.com' ],
315 [8, '8@example.com' ],
316 [9, '9@example.com' ],
317 [10, '10@example.com' ],
318 ], 'msgmap as expected' );
322 local $SIG{__WARN__} = sub { push @warn, @_ };
323 my %config = %$ibx_config;
324 my $ibx = PublicInbox::Inbox->new(\%config);
325 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
326 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
327 # mark4 A delete of an older message
328 eval { $rw->index_sync({ref => $mark4}) };
329 is($@, '', 'no error from reindexing without msgmap');
330 is_deeply(\@warn, [], 'no warnings');
332 my ($min, $max) = $ibx->mm->minmax;
333 is($min, 1, 'min as expected');
334 is($max, 10, 'max as expected');
335 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
336 is_deeply($ibx->mm->msg_range(\$min, $max),
338 [1, '1@example.com' ],
339 [2, '2@example.com' ],
340 [3, '3@example.com' ],
341 [6, '6@example.com' ],
342 [7, '7@example.com' ],
343 [8, '8@example.com' ],
344 [9, '9@example.com' ],
345 [10, '10@example.com' ],
346 ], 'msgmap as expected' );
350 # Another incremental indexing test
351 ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
353 ok(!-d $xap, 'Xapian directories removed again');
356 local $SIG{__WARN__} = sub { push @warn, @_ };
357 my %config = %$ibx_config;
358 my $ibx = PublicInbox::Inbox->new(\%config);
359 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
360 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
361 # mark2 an add and it's delete in the same index_sync
362 eval { $rw->index_sync({ref => $mark2}) };
363 is($@, '', 'no error from reindexing without msgmap');
364 is_deeply(\@warn, [], 'no warnings');
366 my ($min, $max) = $ibx->mm->minmax;
367 is($min, 1, 'min as expected');
368 is($max, 3, 'max as expected');
369 is($ibx->mm->num_highwater, 4, 'num_highwater as expected');
370 is_deeply($ibx->mm->msg_range(\$min, $max),
372 [1, '1@example.com' ],
373 [2, '2@example.com' ],
374 [3, '3@example.com' ],
375 ], 'msgmap as expected' );
379 local $SIG{__WARN__} = sub { push @warn, @_ };
380 my %config = %$ibx_config;
381 my $ibx = PublicInbox::Inbox->new(\%config);
382 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
383 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
384 # mark3 adds following the delete at mark2
385 eval { $rw->index_sync({ref => $mark3}) };
386 is($@, '', 'no error from reindexing without msgmap');
387 is_deeply(\@warn, [], 'no warnings');
389 my ($min, $max) = $ibx->mm->minmax;
390 is($min, 1, 'min as expected');
391 is($max, 10, 'max as expected');
392 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
393 is_deeply($ibx->mm->msg_range(\$min, $max),
395 [1, '1@example.com' ],
396 [2, '2@example.com' ],
397 [3, '3@example.com' ],
398 [5, '5@example.com' ],
399 [6, '6@example.com' ],
400 [7, '7@example.com' ],
401 [8, '8@example.com' ],
402 [9, '9@example.com' ],
403 [10, '10@example.com' ],
404 ], 'msgmap as expected' );
408 local $SIG{__WARN__} = sub { push @warn, @_ };
409 my %config = %$ibx_config;
410 my $ibx = PublicInbox::Inbox->new(\%config);
411 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
412 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
413 # mark4 A delete of an older message
414 eval { $rw->index_sync({ref => $mark4}) };
415 is($@, '', 'no error from reindexing without msgmap');
416 is_deeply(\@warn, [], 'no warnings');
418 my ($min, $max) = $ibx->mm->minmax;
419 is($min, 1, 'min as expected');
420 is($max, 10, 'max as expected');
421 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
422 is_deeply($ibx->mm->msg_range(\$min, $max),
424 [1, '1@example.com' ],
425 [2, '2@example.com' ],
426 [3, '3@example.com' ],
427 [6, '6@example.com' ],
428 [7, '7@example.com' ],
429 [8, '8@example.com' ],
430 [9, '9@example.com' ],
431 [10, '10@example.com' ],
432 ], 'msgmap as expected' );