1 # Copyright (C) 2018-2019 all contributors <meta@public-inbox.org>
2 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
7 use PublicInbox::ContentId qw(content_digest);
8 use File::Temp qw/tempdir/;
9 use File::Path qw(remove_tree);
10 require './t/common.perl';
13 foreach my $mod (qw(DBD::SQLite Search::Xapian)) {
15 plan skip_all => "$mod missing for v1reindex.t" if $@;
17 use_ok 'PublicInbox::SearchIdx';
18 use_ok 'PublicInbox::Import';
19 my $mainrepo = tempdir('pi-v1reindex-XXXXXX', TMPDIR => 1, CLEANUP => 1);
20 is(system(qw(git init -q --bare), $mainrepo), 0);
22 mainrepo => $mainrepo,
23 name => 'test-v1reindex',
24 -primary_address => 'test@example.com',
27 my $mime = PublicInbox::MIME->create(
29 From => 'a@example.com',
30 To => 'test@example.com',
31 Subject => 'this is a subject',
32 Date => 'Fri, 02 Oct 1993 00:00:00 +0000',
34 body => "hello world\n",
38 my ($mark1, $mark2, $mark3, $mark4);
40 my %config = %$ibx_config;
41 my $ibx = PublicInbox::Inbox->new(\%config);
42 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
43 foreach my $i (1..10) {
44 $mime->header_set('Message-Id', "<$i\@example.com>");
45 ok($im->add($mime), "message $i added");
47 $mark1 = $im->get_mark($im->{tip});
49 $mark2 = $im->get_mark($im->{tip});
53 if ('test remove later') {
54 $mark3 = $im->get_mark($im->{tip});
55 $mime->header_set('Message-Id', "<5\@example.com>");
57 $mark4 = $im->get_mark($im->{tip});
61 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
62 eval { $rw->index_sync() };
63 is($@, '', 'no error from indexing');
65 $minmax = [ $ibx->mm->minmax ];
66 ok(defined $minmax->[0] && defined $minmax->[1], 'minmax defined');
67 is_deeply($minmax, [ 1, 10 ], 'minmax as expected');
68 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
70 my ($min, $max) = @$minmax;
71 $msgmap = $ibx->mm->msg_range(\$min, $max);
73 [1, '1@example.com' ],
74 [2, '2@example.com' ],
75 [3, '3@example.com' ],
76 [6, '6@example.com' ],
77 [7, '7@example.com' ],
78 [8, '8@example.com' ],
79 [9, '9@example.com' ],
80 [10, '10@example.com' ],
81 ], 'msgmap as expected');
85 my %config = %$ibx_config;
86 my $ibx = PublicInbox::Inbox->new(\%config);
87 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
88 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
89 eval { $rw->index_sync({reindex => 1}) };
90 is($@, '', 'no error from reindexing');
93 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
95 my ($min, $max) = $ibx->mm->minmax;
96 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
99 my $xap = "$mainrepo/public-inbox/xapian".PublicInbox::Search::SCHEMA_VERSION();
101 ok(!-d $xap, 'Xapian directories removed');
103 my %config = %$ibx_config;
104 my $ibx = PublicInbox::Inbox->new(\%config);
105 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
106 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
108 eval { $rw->index_sync({reindex => 1}) };
109 is($@, '', 'no error from reindexing');
111 ok(-d $xap, 'Xapian directories recreated');
114 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
115 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
117 my ($min, $max) = $ibx->mm->minmax;
118 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
121 ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
123 ok(!-d $xap, 'Xapian directories removed again');
126 local $SIG{__WARN__} = sub { push @warn, @_ };
127 my %config = %$ibx_config;
128 my $ibx = PublicInbox::Inbox->new(\%config);
129 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
130 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
131 eval { $rw->index_sync({reindex => 1}) };
132 is($@, '', 'no error from reindexing without msgmap');
133 is(scalar(@warn), 0, 'no warnings from reindexing');
135 ok(-d $xap, 'Xapian directories recreated');
137 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
138 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
140 my ($min, $max) = $ibx->mm->minmax;
141 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
144 ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
146 ok(!-d $xap, 'Xapian directories removed again');
149 local $SIG{__WARN__} = sub { push @warn, @_ };
150 my %config = %$ibx_config;
151 my $ibx = PublicInbox::Inbox->new(\%config);
152 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
153 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
154 eval { $rw->index_sync({reindex => 1}) };
155 is($@, '', 'no error from reindexing without msgmap');
156 is_deeply(\@warn, [], 'no warnings');
158 ok(-d $xap, 'Xapian directories recreated');
160 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
161 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
163 my ($min, $max) = @$minmax;
164 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
167 ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
169 ok(!-d $xap, 'Xapian directories removed again');
172 local $SIG{__WARN__} = sub { push @warn, @_ };
173 my %config = %$ibx_config;
174 $config{indexlevel} = 'medium';
175 my $ibx = PublicInbox::Inbox->new(\%config);
176 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
177 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
178 eval { $rw->index_sync({reindex => 1}) };
179 is($@, '', 'no error from reindexing without msgmap');
180 is_deeply(\@warn, [], 'no warnings');
182 ok(-d $xap, 'Xapian directories recreated');
184 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
185 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
186 my $mset = $ibx->search->query('hello world', {mset=>1});
187 isnt($mset->size, 0, 'got Xapian search results');
189 my ($min, $max) = $ibx->mm->minmax;
190 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
193 ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
195 ok(!-d $xap, 'Xapian directories removed again');
198 local $SIG{__WARN__} = sub { push @warn, @_ };
199 my %config = %$ibx_config;
200 $config{indexlevel} = 'basic';
201 my $ibx = PublicInbox::Inbox->new(\%config);
202 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
203 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
204 eval { $rw->index_sync({reindex => 1}) };
205 is($@, '', 'no error from reindexing without msgmap');
206 is_deeply(\@warn, [], 'no warnings');
208 ok(-d $xap, 'Xapian directories recreated');
210 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
211 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
212 isnt($ibx->search, 'no search for basic');
214 my ($min, $max) = $ibx->mm->minmax;
215 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
218 # upgrade existing basic to medium
219 # note: changing indexlevels is not yet supported in v2,
220 # and may not be without more effort
224 local $SIG{__WARN__} = sub { push @warn, @_ };
225 my %config = %$ibx_config;
226 $config{indexlevel} = 'medium';
227 my $ibx = PublicInbox::Inbox->new(\%config);
228 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
229 eval { $rw->index_sync };
230 is($@, '', 'no error from indexing');
231 is_deeply(\@warn, [], 'no warnings');
232 my $mset = $ibx->search->reopen->query('hello world', {mset=>1});
233 isnt($mset->size, 0, 'search OK after basic -> medium');
235 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
237 my ($min, $max) = $ibx->mm->minmax;
238 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
241 # An incremental indexing test
242 ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
244 ok(!-d $xap, 'Xapian directories removed again');
247 local $SIG{__WARN__} = sub { push @warn, @_ };
248 my %config = %$ibx_config;
249 my $ibx = PublicInbox::Inbox->new(\%config);
250 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
251 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
252 # mark1 4 simple additions in the same index_sync
253 eval { $rw->index_sync({ref => $mark1}) };
254 is($@, '', 'no error from reindexing without msgmap');
255 is_deeply(\@warn, [], 'no warnings');
257 my ($min, $max) = $ibx->mm->minmax;
258 is($min, 1, 'min as expected');
259 is($max, 4, 'max as expected');
260 is($ibx->mm->num_highwater, 4, 'num_highwater as expected');
261 is_deeply($ibx->mm->msg_range(\$min, $max),
263 [1, '1@example.com' ],
264 [2, '2@example.com' ],
265 [3, '3@example.com' ],
266 [4, '4@example.com' ],
267 ], 'msgmap as expected' );
271 local $SIG{__WARN__} = sub { push @warn, @_ };
272 my %config = %$ibx_config;
273 my $ibx = PublicInbox::Inbox->new(\%config);
274 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
275 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
276 # mark2 A delete separated form and add in the same index_sync
277 eval { $rw->index_sync({ref => $mark2}) };
278 is($@, '', 'no error from reindexing without msgmap');
279 is_deeply(\@warn, [], 'no warnings');
281 my ($min, $max) = $ibx->mm->minmax;
282 is($min, 1, 'min as expected');
283 is($max, 3, 'max as expected');
284 is($ibx->mm->num_highwater, 4, 'num_highwater as expected');
285 is_deeply($ibx->mm->msg_range(\$min, $max),
287 [1, '1@example.com' ],
288 [2, '2@example.com' ],
289 [3, '3@example.com' ],
290 ], 'msgmap as expected' );
294 local $SIG{__WARN__} = sub { push @warn, @_ };
295 my %config = %$ibx_config;
296 my $ibx = PublicInbox::Inbox->new(\%config);
297 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
298 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
299 # mark3 adds following the delete at mark2
300 eval { $rw->index_sync({ref => $mark3}) };
301 is($@, '', 'no error from reindexing without msgmap');
302 is_deeply(\@warn, [], 'no warnings');
304 my ($min, $max) = $ibx->mm->minmax;
305 is($min, 1, 'min as expected');
306 is($max, 10, 'max as expected');
307 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
308 is_deeply($ibx->mm->msg_range(\$min, $max),
310 [1, '1@example.com' ],
311 [2, '2@example.com' ],
312 [3, '3@example.com' ],
313 [5, '5@example.com' ],
314 [6, '6@example.com' ],
315 [7, '7@example.com' ],
316 [8, '8@example.com' ],
317 [9, '9@example.com' ],
318 [10, '10@example.com' ],
319 ], 'msgmap as expected' );
323 local $SIG{__WARN__} = sub { push @warn, @_ };
324 my %config = %$ibx_config;
325 my $ibx = PublicInbox::Inbox->new(\%config);
326 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
327 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
328 # mark4 A delete of an older message
329 eval { $rw->index_sync({ref => $mark4}) };
330 is($@, '', 'no error from reindexing without msgmap');
331 is_deeply(\@warn, [], 'no warnings');
333 my ($min, $max) = $ibx->mm->minmax;
334 is($min, 1, 'min as expected');
335 is($max, 10, 'max as expected');
336 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
337 is_deeply($ibx->mm->msg_range(\$min, $max),
339 [1, '1@example.com' ],
340 [2, '2@example.com' ],
341 [3, '3@example.com' ],
342 [6, '6@example.com' ],
343 [7, '7@example.com' ],
344 [8, '8@example.com' ],
345 [9, '9@example.com' ],
346 [10, '10@example.com' ],
347 ], 'msgmap as expected' );
351 # Another incremental indexing test
352 ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
354 ok(!-d $xap, 'Xapian directories removed again');
357 local $SIG{__WARN__} = sub { push @warn, @_ };
358 my %config = %$ibx_config;
359 my $ibx = PublicInbox::Inbox->new(\%config);
360 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
361 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
362 # mark2 an add and it's delete in the same index_sync
363 eval { $rw->index_sync({ref => $mark2}) };
364 is($@, '', 'no error from reindexing without msgmap');
365 is_deeply(\@warn, [], 'no warnings');
367 my ($min, $max) = $ibx->mm->minmax;
368 is($min, 1, 'min as expected');
369 is($max, 3, 'max as expected');
370 is($ibx->mm->num_highwater, 4, 'num_highwater as expected');
371 is_deeply($ibx->mm->msg_range(\$min, $max),
373 [1, '1@example.com' ],
374 [2, '2@example.com' ],
375 [3, '3@example.com' ],
376 ], 'msgmap as expected' );
380 local $SIG{__WARN__} = sub { push @warn, @_ };
381 my %config = %$ibx_config;
382 my $ibx = PublicInbox::Inbox->new(\%config);
383 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
384 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
385 # mark3 adds following the delete at mark2
386 eval { $rw->index_sync({ref => $mark3}) };
387 is($@, '', 'no error from reindexing without msgmap');
388 is_deeply(\@warn, [], 'no warnings');
390 my ($min, $max) = $ibx->mm->minmax;
391 is($min, 1, 'min as expected');
392 is($max, 10, 'max as expected');
393 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
394 is_deeply($ibx->mm->msg_range(\$min, $max),
396 [1, '1@example.com' ],
397 [2, '2@example.com' ],
398 [3, '3@example.com' ],
399 [5, '5@example.com' ],
400 [6, '6@example.com' ],
401 [7, '7@example.com' ],
402 [8, '8@example.com' ],
403 [9, '9@example.com' ],
404 [10, '10@example.com' ],
405 ], 'msgmap as expected' );
409 local $SIG{__WARN__} = sub { push @warn, @_ };
410 my %config = %$ibx_config;
411 my $ibx = PublicInbox::Inbox->new(\%config);
412 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
413 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
414 # mark4 A delete of an older message
415 eval { $rw->index_sync({ref => $mark4}) };
416 is($@, '', 'no error from reindexing without msgmap');
417 is_deeply(\@warn, [], 'no warnings');
419 my ($min, $max) = $ibx->mm->minmax;
420 is($min, 1, 'min as expected');
421 is($max, 10, 'max as expected');
422 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
423 is_deeply($ibx->mm->msg_range(\$min, $max),
425 [1, '1@example.com' ],
426 [2, '2@example.com' ],
427 [3, '3@example.com' ],
428 [6, '6@example.com' ],
429 [7, '7@example.com' ],
430 [8, '8@example.com' ],
431 [9, '9@example.com' ],
432 [10, '10@example.com' ],
433 ], 'msgmap as expected' );