1 # Copyright (C) 2018 all contributors <meta@public-inbox.org>
2 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
7 use PublicInbox::ContentId qw(content_digest);
8 use File::Temp qw/tempdir/;
9 use File::Path qw(remove_tree);
11 foreach my $mod (qw(DBD::SQLite Search::Xapian)) {
13 plan skip_all => "$mod missing for v1reindex.t" if $@;
15 use_ok 'PublicInbox::SearchIdx';
16 use_ok 'PublicInbox::Import';
17 my $mainrepo = tempdir('pi-v1reindex-XXXXXX', TMPDIR => 1, CLEANUP => 1);
18 is(system(qw(git init -q --bare), $mainrepo), 0);
20 mainrepo => $mainrepo,
21 name => 'test-v1reindex',
22 -primary_address => 'test@example.com',
25 my $mime = PublicInbox::MIME->create(
27 From => 'a@example.com',
28 To => 'test@example.com',
29 Subject => 'this is a subject',
30 Date => 'Fri, 02 Oct 1993 00:00:00 +0000',
32 body => "hello world\n",
36 my ($mark1, $mark2, $mark3, $mark4);
38 my %config = %$ibx_config;
39 my $ibx = PublicInbox::Inbox->new(\%config);
40 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
41 foreach my $i (1..10) {
42 $mime->header_set('Message-Id', "<$i\@example.com>");
43 ok($im->add($mime), "message $i added");
45 $mark1 = $im->get_mark($im->{tip});
47 $mark2 = $im->get_mark($im->{tip});
51 if ('test remove later') {
52 $mark3 = $im->get_mark($im->{tip});
53 $mime->header_set('Message-Id', "<5\@example.com>");
55 $mark4 = $im->get_mark($im->{tip});
59 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
60 eval { $rw->index_sync() };
61 is($@, '', 'no error from indexing');
63 $minmax = [ $ibx->mm->minmax ];
64 ok(defined $minmax->[0] && defined $minmax->[1], 'minmax defined');
65 is_deeply($minmax, [ 1, 10 ], 'minmax as expected');
67 my ($min, $max) = @$minmax;
68 $msgmap = $ibx->mm->msg_range(\$min, $max);
70 [1, '1@example.com' ],
71 [2, '2@example.com' ],
72 [3, '3@example.com' ],
73 [6, '6@example.com' ],
74 [7, '7@example.com' ],
75 [8, '8@example.com' ],
76 [9, '9@example.com' ],
77 [10, '10@example.com' ],
78 ], 'msgmap as expected');
82 my %config = %$ibx_config;
83 my $ibx = PublicInbox::Inbox->new(\%config);
84 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
85 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
86 eval { $rw->index_sync({reindex => 1}) };
87 is($@, '', 'no error from reindexing');
90 my ($min, $max) = $ibx->mm->minmax;
91 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
94 my $xap = "$mainrepo/public-inbox/xapian".PublicInbox::Search::SCHEMA_VERSION();
96 ok(!-d $xap, 'Xapian directories removed');
98 my %config = %$ibx_config;
99 my $ibx = PublicInbox::Inbox->new(\%config);
100 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
101 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
103 eval { $rw->index_sync({reindex => 1}) };
104 is($@, '', 'no error from reindexing');
106 ok(-d $xap, 'Xapian directories recreated');
109 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
111 my ($min, $max) = $ibx->mm->minmax;
112 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
115 ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
117 ok(!-d $xap, 'Xapian directories removed again');
120 local $SIG{__WARN__} = sub { push @warn, @_ };
121 my %config = %$ibx_config;
122 my $ibx = PublicInbox::Inbox->new(\%config);
123 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
124 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
125 eval { $rw->index_sync({reindex => 1}) };
126 is($@, '', 'no error from reindexing without msgmap');
127 is(scalar(@warn), 0, 'no warnings from reindexing');
129 ok(-d $xap, 'Xapian directories recreated');
131 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
133 my ($min, $max) = $ibx->mm->minmax;
134 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
137 ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
139 ok(!-d $xap, 'Xapian directories removed again');
142 local $SIG{__WARN__} = sub { push @warn, @_ };
143 my %config = %$ibx_config;
144 my $ibx = PublicInbox::Inbox->new(\%config);
145 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
146 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
147 eval { $rw->index_sync({reindex => 1}) };
148 is($@, '', 'no error from reindexing without msgmap');
149 is_deeply(\@warn, [], 'no warnings');
151 ok(-d $xap, 'Xapian directories recreated');
153 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
155 my ($min, $max) = @$minmax;
156 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
159 ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
161 ok(!-d $xap, 'Xapian directories removed again');
164 local $SIG{__WARN__} = sub { push @warn, @_ };
165 my %config = %$ibx_config;
166 $config{indexlevel} = 'medium';
167 my $ibx = PublicInbox::Inbox->new(\%config);
168 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
169 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
170 eval { $rw->index_sync({reindex => 1}) };
171 is($@, '', 'no error from reindexing without msgmap');
172 is_deeply(\@warn, [], 'no warnings');
174 ok(-d $xap, 'Xapian directories recreated');
176 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
177 my $mset = $ibx->search->query('hello world', {mset=>1});
178 isnt($mset->size, 0, 'got Xapian search results');
180 my ($min, $max) = $ibx->mm->minmax;
181 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
184 ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
186 ok(!-d $xap, 'Xapian directories removed again');
189 local $SIG{__WARN__} = sub { push @warn, @_ };
190 my %config = %$ibx_config;
191 $config{indexlevel} = 'basic';
192 my $ibx = PublicInbox::Inbox->new(\%config);
193 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
194 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
195 eval { $rw->index_sync({reindex => 1}) };
196 is($@, '', 'no error from reindexing without msgmap');
197 is_deeply(\@warn, [], 'no warnings');
199 ok(-d $xap, 'Xapian directories recreated');
201 is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
202 my $mset = $ibx->search->reopen->query('hello world', {mset=>1});
203 is($mset->size, 0, "no Xapian search results");
205 my ($min, $max) = $ibx->mm->minmax;
206 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
209 # upgrade existing basic to medium
210 # note: changing indexlevels is not yet supported in v2,
211 # and may not be without more effort
215 local $SIG{__WARN__} = sub { push @warn, @_ };
216 my %config = %$ibx_config;
217 $config{indexleve} = 'medium';
218 my $ibx = PublicInbox::Inbox->new(\%config);
219 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
220 eval { $rw->index_sync };
221 is($@, '', 'no error from indexing');
222 is_deeply(\@warn, [], 'no warnings');
223 my $mset = $ibx->search->reopen->query('hello world', {mset=>1});
224 isnt($mset->size, 0, 'search OK after basic -> medium');
226 my ($min, $max) = $ibx->mm->minmax;
227 is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
230 # An incremental indexing test
231 ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
233 ok(!-d $xap, 'Xapian directories removed again');
236 local $SIG{__WARN__} = sub { push @warn, @_ };
237 my %config = %$ibx_config;
238 my $ibx = PublicInbox::Inbox->new(\%config);
239 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
240 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
241 # mark1 4 simple additions in the same index_sync
242 eval { $rw->index_sync({ref => $mark1}) };
243 is($@, '', 'no error from reindexing without msgmap');
244 is_deeply(\@warn, [], 'no warnings');
246 my ($min, $max) = $ibx->mm->minmax;
247 is($min, 1, 'min as expected');
248 is($max, 4, 'max as expected');
249 is_deeply($ibx->mm->msg_range(\$min, $max),
251 [1, '1@example.com' ],
252 [2, '2@example.com' ],
253 [3, '3@example.com' ],
254 [4, '4@example.com' ],
255 ], 'msgmap as expected' );
259 local $SIG{__WARN__} = sub { push @warn, @_ };
260 my %config = %$ibx_config;
261 my $ibx = PublicInbox::Inbox->new(\%config);
262 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
263 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
264 # mark2 A delete separated form and add in the same index_sync
265 eval { $rw->index_sync({ref => $mark2}) };
266 is($@, '', 'no error from reindexing without msgmap');
267 is_deeply(\@warn, [], 'no warnings');
269 my ($min, $max) = $ibx->mm->minmax;
270 is($min, 1, 'min as expected');
271 is($max, 3, 'max as expected');
272 is_deeply($ibx->mm->msg_range(\$min, $max),
274 [1, '1@example.com' ],
275 [2, '2@example.com' ],
276 [3, '3@example.com' ],
277 ], 'msgmap as expected' );
281 local $SIG{__WARN__} = sub { push @warn, @_ };
282 my %config = %$ibx_config;
283 my $ibx = PublicInbox::Inbox->new(\%config);
284 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
285 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
286 # mark3 adds following the delete at mark2
287 eval { $rw->index_sync({ref => $mark3}) };
288 is($@, '', 'no error from reindexing without msgmap');
289 is_deeply(\@warn, [], 'no warnings');
291 my ($min, $max) = $ibx->mm->minmax;
292 is($min, 1, 'min as expected');
293 is($max, 10, 'max as expected');
294 is_deeply($ibx->mm->msg_range(\$min, $max),
296 [1, '1@example.com' ],
297 [2, '2@example.com' ],
298 [3, '3@example.com' ],
299 [5, '5@example.com' ],
300 [6, '6@example.com' ],
301 [7, '7@example.com' ],
302 [8, '8@example.com' ],
303 [9, '9@example.com' ],
304 [10, '10@example.com' ],
305 ], 'msgmap as expected' );
309 local $SIG{__WARN__} = sub { push @warn, @_ };
310 my %config = %$ibx_config;
311 my $ibx = PublicInbox::Inbox->new(\%config);
312 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
313 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
314 # mark4 A delete of an older message
315 eval { $rw->index_sync({ref => $mark4}) };
316 is($@, '', 'no error from reindexing without msgmap');
317 is_deeply(\@warn, [], 'no warnings');
319 my ($min, $max) = $ibx->mm->minmax;
320 is($min, 1, 'min as expected');
321 is($max, 10, 'max as expected');
322 is_deeply($ibx->mm->msg_range(\$min, $max),
324 [1, '1@example.com' ],
325 [2, '2@example.com' ],
326 [3, '3@example.com' ],
327 [6, '6@example.com' ],
328 [7, '7@example.com' ],
329 [8, '8@example.com' ],
330 [9, '9@example.com' ],
331 [10, '10@example.com' ],
332 ], 'msgmap as expected' );
336 # Another incremental indexing test
337 ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
339 ok(!-d $xap, 'Xapian directories removed again');
342 local $SIG{__WARN__} = sub { push @warn, @_ };
343 my %config = %$ibx_config;
344 my $ibx = PublicInbox::Inbox->new(\%config);
345 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
346 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
347 # mark2 an add and it's delete in the same index_sync
348 eval { $rw->index_sync({ref => $mark2}) };
349 is($@, '', 'no error from reindexing without msgmap');
350 is_deeply(\@warn, [], 'no warnings');
352 my ($min, $max) = $ibx->mm->minmax;
353 is($min, 1, 'min as expected');
354 is($max, 3, 'max as expected');
355 is_deeply($ibx->mm->msg_range(\$min, $max),
357 [1, '1@example.com' ],
358 [2, '2@example.com' ],
359 [3, '3@example.com' ],
360 ], 'msgmap as expected' );
364 local $SIG{__WARN__} = sub { push @warn, @_ };
365 my %config = %$ibx_config;
366 my $ibx = PublicInbox::Inbox->new(\%config);
367 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
368 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
369 # mark3 adds following the delete at mark2
370 eval { $rw->index_sync({ref => $mark3}) };
371 is($@, '', 'no error from reindexing without msgmap');
372 is_deeply(\@warn, [], 'no warnings');
374 my ($min, $max) = $ibx->mm->minmax;
375 is($min, 1, 'min as expected');
376 is($max, 10, 'max as expected');
377 is_deeply($ibx->mm->msg_range(\$min, $max),
379 [1, '1@example.com' ],
380 [2, '2@example.com' ],
381 [3, '3@example.com' ],
382 [5, '5@example.com' ],
383 [6, '6@example.com' ],
384 [7, '7@example.com' ],
385 [8, '8@example.com' ],
386 [9, '9@example.com' ],
387 [10, '10@example.com' ],
388 ], 'msgmap as expected' );
392 local $SIG{__WARN__} = sub { push @warn, @_ };
393 my %config = %$ibx_config;
394 my $ibx = PublicInbox::Inbox->new(\%config);
395 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
396 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
397 # mark4 A delete of an older message
398 eval { $rw->index_sync({ref => $mark4}) };
399 is($@, '', 'no error from reindexing without msgmap');
400 is_deeply(\@warn, [], 'no warnings');
402 my ($min, $max) = $ibx->mm->minmax;
403 is($min, 1, 'min as expected');
404 is($max, 10, 'max as expected');
405 is_deeply($ibx->mm->msg_range(\$min, $max),
407 [1, '1@example.com' ],
408 [2, '2@example.com' ],
409 [3, '3@example.com' ],
410 [6, '6@example.com' ],
411 [7, '7@example.com' ],
412 [8, '8@example.com' ],
413 [9, '9@example.com' ],
414 [10, '10@example.com' ],
415 ], 'msgmap as expected' );