1 # Copyright (C) 2018 all contributors <meta@public-inbox.org>
2 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
7 use PublicInbox::ContentId qw(content_digest);
8 use File::Temp qw/tempdir/;
9 use File::Path qw(remove_tree);
11 foreach my $mod (qw(DBD::SQLite Search::Xapian)) {
13 plan skip_all => "$mod missing for v1reindex.t" if $@;
15 use_ok 'PublicInbox::SearchIdx';
16 use_ok 'PublicInbox::Import';
17 my $mainrepo = tempdir('pi-v1reindex-XXXXXX', TMPDIR => 1, CLEANUP => 1);
18 is(system(qw(git init -q --bare), $mainrepo), 0);
20 mainrepo => $mainrepo,
21 name => 'test-v1reindex',
22 -primary_address => 'test@example.com',
25 my $mime = PublicInbox::MIME->create(
27 From => 'a@example.com',
28 To => 'test@example.com',
29 Subject => 'this is a subject',
30 Date => 'Fri, 02 Oct 1993 00:00:00 +0000',
32 body => "hello world\n",
36 my ($mark1, $mark2, $mark3, $mark4);
38 my %config = %$ibx_config;
39 my $ibx = PublicInbox::Inbox->new(\%config);
40 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
41 foreach my $i (1..10) {
42 $mime->header_set('Message-Id', "<$i\@example.com>");
43 ok($im->add($mime), "message $i added");
45 $mark1 = $im->get_mark($im->{tip});
47 $mark2 = $im->get_mark($im->{tip});
51 if ('test remove later') {
52 $mark3 = $im->get_mark($im->{tip});
53 $mime->header_set('Message-Id', "<5\@example.com>");
55 $mark4 = $im->get_mark($im->{tip});
59 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
60 eval { $rw->index_sync() };
61 is($@, '', 'no error from indexing');
63 $minmax = [ $ibx->mm->minmax ];
64 ok(defined $minmax->[0] && defined $minmax->[1], 'minmax defined');
65 is_deeply($minmax, [ 1, 10 ], 'minmax as expected');
67 my ($min, $max) = @$minmax;
68 $msgmap = $ibx->mm->msg_range(\$min, $max);
70 [1, '1@example.com' ],
71 [2, '2@example.com' ],
72 [3, '3@example.com' ],
73 [6, '6@example.com' ],
74 [7, '7@example.com' ],
75 [8, '8@example.com' ],
76 [9, '9@example.com' ],
77 [10, '10@example.com' ],
78 ], 'msgmap as expected');
82 my %config = %$ibx_config;
83 my $ibx = PublicInbox::Inbox->new(\%config);
84 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
85 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
86 eval { $rw->index_sync({reindex => 1}) };
87 is($@, '', 'no error from reindexing');
90 my ($min, $max) = $ibx->mm->minmax;
91 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
94 my $xap = "$mainrepo/public-inbox/xapian".PublicInbox::Search::SCHEMA_VERSION();
96 ok(!-d $xap, 'Xapian directories removed');
98 my %config = %$ibx_config;
99 my $ibx = PublicInbox::Inbox->new(\%config);
100 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
101 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
103 eval { $rw->index_sync({reindex => 1}) };
104 is($@, '', 'no error from reindexing');
106 ok(-d $xap, 'Xapian directories recreated');
109 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
111 my ($min, $max) = $ibx->mm->minmax;
112 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
115 ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
117 ok(!-d $xap, 'Xapian directories removed again');
120 local $SIG{__WARN__} = sub { push @warn, @_ };
121 my %config = %$ibx_config;
122 my $ibx = PublicInbox::Inbox->new(\%config);
123 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
124 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
125 eval { $rw->index_sync({reindex => 1}) };
126 is($@, '', 'no error from reindexing without msgmap');
127 is(scalar(@warn), 0, 'no warnings from reindexing');
129 ok(-d $xap, 'Xapian directories recreated');
131 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
133 my ($min, $max) = $ibx->mm->minmax;
134 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
137 ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
139 ok(!-d $xap, 'Xapian directories removed again');
142 local $SIG{__WARN__} = sub { push @warn, @_ };
143 my %config = %$ibx_config;
144 my $ibx = PublicInbox::Inbox->new(\%config);
145 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
146 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
147 eval { $rw->index_sync({reindex => 1}) };
148 is($@, '', 'no error from reindexing without msgmap');
149 is_deeply(\@warn, [], 'no warnings');
151 ok(-d $xap, 'Xapian directories recreated');
153 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
155 my ($min, $max) = @$minmax;
156 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
159 ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
161 ok(!-d $xap, 'Xapian directories removed again');
164 local $SIG{__WARN__} = sub { push @warn, @_ };
165 my %config = %$ibx_config;
166 $config{indexlevel} = 'medium';
167 my $ibx = PublicInbox::Inbox->new(\%config);
168 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
169 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
170 eval { $rw->index_sync({reindex => 1}) };
171 is($@, '', 'no error from reindexing without msgmap');
172 is_deeply(\@warn, [], 'no warnings');
174 ok(-d $xap, 'Xapian directories recreated');
176 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
177 my $mset = $ibx->search->query('hello world', {mset=>1});
178 isnt($mset->size, 0, 'got Xapian search results');
180 my ($min, $max) = $ibx->mm->minmax;
181 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
184 ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
186 ok(!-d $xap, 'Xapian directories removed again');
189 local $SIG{__WARN__} = sub { push @warn, @_ };
190 my %config = %$ibx_config;
191 $config{indexlevel} = 'basic';
192 my $ibx = PublicInbox::Inbox->new(\%config);
193 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
194 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
195 eval { $rw->index_sync({reindex => 1}) };
196 is($@, '', 'no error from reindexing without msgmap');
197 is_deeply(\@warn, [], 'no warnings');
199 ok(-d $xap, 'Xapian directories recreated');
201 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
202 my $mset = $ibx->search->reopen->query('hello world', {mset=>1});
203 is($mset->size, 0, "no Xapian search results");
205 my ($min, $max) = $ibx->mm->minmax;
206 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
209 # upgrade existing basic to medium
210 # note: changing indexlevels is not yet supported in v2,
211 # and may not be without more effort
215 local $SIG{__WARN__} = sub { push @warn, @_ };
216 my %config = %$ibx_config;
217 $config{indexleve} = 'medium';
218 my $ibx = PublicInbox::Inbox->new(\%config);
219 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
220 eval { $rw->index_sync };
221 is($@, '', 'no error from indexing');
222 is_deeply(\@warn, [], 'no warnings');
223 my $mset = $ibx->search->reopen->query('hello world', {mset=>1});
224 isnt($mset->size, 0, 'search OK after basic -> medium');
226 my ($min, $max) = $ibx->mm->minmax;
227 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
230 # An incremental indexing test
231 ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
233 ok(!-d $xap, 'Xapian directories removed again');
236 local $SIG{__WARN__} = sub { push @warn, @_ };
237 my %config = %$ibx_config;
238 my $ibx = PublicInbox::Inbox->new(\%config);
239 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
240 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
241 # mark1 4 simple additions in the same index_sync
242 eval { $rw->index_sync({ref => $mark1}) };
243 is($@, '', 'no error from reindexing without msgmap');
244 is_deeply(\@warn, [], 'no warnings');
246 my ($min, $max) = $ibx->mm->minmax;
247 is($min, 1, 'min as expected');
248 is($max, 4, 'max as expected');
249 is($ibx->mm->num_highwater, 4, 'num_highwater as expected');
250 is_deeply($ibx->mm->msg_range(\$min, $max),
252 [1, '1@example.com' ],
253 [2, '2@example.com' ],
254 [3, '3@example.com' ],
255 [4, '4@example.com' ],
256 ], 'msgmap as expected' );
260 local $SIG{__WARN__} = sub { push @warn, @_ };
261 my %config = %$ibx_config;
262 my $ibx = PublicInbox::Inbox->new(\%config);
263 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
264 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
265 # mark2 A delete separated form and add in the same index_sync
266 eval { $rw->index_sync({ref => $mark2}) };
267 is($@, '', 'no error from reindexing without msgmap');
268 is_deeply(\@warn, [], 'no warnings');
270 my ($min, $max) = $ibx->mm->minmax;
271 is($min, 1, 'min as expected');
272 is($max, 3, 'max as expected');
273 is($ibx->mm->num_highwater, 4, 'num_highwater as expected');
274 is_deeply($ibx->mm->msg_range(\$min, $max),
276 [1, '1@example.com' ],
277 [2, '2@example.com' ],
278 [3, '3@example.com' ],
279 ], 'msgmap as expected' );
283 local $SIG{__WARN__} = sub { push @warn, @_ };
284 my %config = %$ibx_config;
285 my $ibx = PublicInbox::Inbox->new(\%config);
286 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
287 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
288 # mark3 adds following the delete at mark2
289 eval { $rw->index_sync({ref => $mark3}) };
290 is($@, '', 'no error from reindexing without msgmap');
291 is_deeply(\@warn, [], 'no warnings');
293 my ($min, $max) = $ibx->mm->minmax;
294 is($min, 1, 'min as expected');
295 is($max, 10, 'max as expected');
296 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
297 is_deeply($ibx->mm->msg_range(\$min, $max),
299 [1, '1@example.com' ],
300 [2, '2@example.com' ],
301 [3, '3@example.com' ],
302 [5, '5@example.com' ],
303 [6, '6@example.com' ],
304 [7, '7@example.com' ],
305 [8, '8@example.com' ],
306 [9, '9@example.com' ],
307 [10, '10@example.com' ],
308 ], 'msgmap as expected' );
312 local $SIG{__WARN__} = sub { push @warn, @_ };
313 my %config = %$ibx_config;
314 my $ibx = PublicInbox::Inbox->new(\%config);
315 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
316 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
317 # mark4 A delete of an older message
318 eval { $rw->index_sync({ref => $mark4}) };
319 is($@, '', 'no error from reindexing without msgmap');
320 is_deeply(\@warn, [], 'no warnings');
322 my ($min, $max) = $ibx->mm->minmax;
323 is($min, 1, 'min as expected');
324 is($max, 10, 'max as expected');
325 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
326 is_deeply($ibx->mm->msg_range(\$min, $max),
328 [1, '1@example.com' ],
329 [2, '2@example.com' ],
330 [3, '3@example.com' ],
331 [6, '6@example.com' ],
332 [7, '7@example.com' ],
333 [8, '8@example.com' ],
334 [9, '9@example.com' ],
335 [10, '10@example.com' ],
336 ], 'msgmap as expected' );
340 # Another incremental indexing test
341 ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
343 ok(!-d $xap, 'Xapian directories removed again');
346 local $SIG{__WARN__} = sub { push @warn, @_ };
347 my %config = %$ibx_config;
348 my $ibx = PublicInbox::Inbox->new(\%config);
349 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
350 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
351 # mark2 an add and it's delete in the same index_sync
352 eval { $rw->index_sync({ref => $mark2}) };
353 is($@, '', 'no error from reindexing without msgmap');
354 is_deeply(\@warn, [], 'no warnings');
356 my ($min, $max) = $ibx->mm->minmax;
357 is($min, 1, 'min as expected');
358 is($max, 3, 'max as expected');
359 is($ibx->mm->num_highwater, 4, 'num_highwater as expected');
360 is_deeply($ibx->mm->msg_range(\$min, $max),
362 [1, '1@example.com' ],
363 [2, '2@example.com' ],
364 [3, '3@example.com' ],
365 ], 'msgmap as expected' );
369 local $SIG{__WARN__} = sub { push @warn, @_ };
370 my %config = %$ibx_config;
371 my $ibx = PublicInbox::Inbox->new(\%config);
372 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
373 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
374 # mark3 adds following the delete at mark2
375 eval { $rw->index_sync({ref => $mark3}) };
376 is($@, '', 'no error from reindexing without msgmap');
377 is_deeply(\@warn, [], 'no warnings');
379 my ($min, $max) = $ibx->mm->minmax;
380 is($min, 1, 'min as expected');
381 is($max, 10, 'max as expected');
382 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
383 is_deeply($ibx->mm->msg_range(\$min, $max),
385 [1, '1@example.com' ],
386 [2, '2@example.com' ],
387 [3, '3@example.com' ],
388 [5, '5@example.com' ],
389 [6, '6@example.com' ],
390 [7, '7@example.com' ],
391 [8, '8@example.com' ],
392 [9, '9@example.com' ],
393 [10, '10@example.com' ],
394 ], 'msgmap as expected' );
398 local $SIG{__WARN__} = sub { push @warn, @_ };
399 my %config = %$ibx_config;
400 my $ibx = PublicInbox::Inbox->new(\%config);
401 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
402 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
403 # mark4 A delete of an older message
404 eval { $rw->index_sync({ref => $mark4}) };
405 is($@, '', 'no error from reindexing without msgmap');
406 is_deeply(\@warn, [], 'no warnings');
408 my ($min, $max) = $ibx->mm->minmax;
409 is($min, 1, 'min as expected');
410 is($max, 10, 'max as expected');
411 is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
412 is_deeply($ibx->mm->msg_range(\$min, $max),
414 [1, '1@example.com' ],
415 [2, '2@example.com' ],
416 [3, '3@example.com' ],
417 [6, '6@example.com' ],
418 [7, '7@example.com' ],
419 [8, '8@example.com' ],
420 [9, '9@example.com' ],
421 [10, '10@example.com' ],
422 ], 'msgmap as expected' );