extindex: -xcpdb and -compact support

[public-inbox.git] / t / extsearch.t

diff --git a/t/extsearch.t b/t/extsearch.t
index 2c3f7547f76f2e17118326b75271c5fc4cb27c37..d933b9485281791745f2df2854c9d1802aab003b 100644 (file)
--- a/t/extsearch.t
+++ b/t/extsearch.t
@@ -5,16 +5,16 @@ use strict;
 use Test::More;
 use PublicInbox::TestCommon;
 use PublicInbox::Config;
-use PublicInbox::Search;
 use PublicInbox::InboxWritable;
 use Fcntl qw(:seek);
 require_git(2.6);
 require_mods(qw(json DBD::SQLite Search::Xapian));
+require PublicInbox::Search;
 use_ok 'PublicInbox::ExtSearch';
 use_ok 'PublicInbox::ExtSearchIdx';
 use_ok 'PublicInbox::OverIdx';
 my $sock = tcp_server();
-my $host_port = $sock->sockhost . ':' . $sock->sockport;
+my $host_port = tcp_host_port($sock);
 my ($home, $for_destroy) = tmpdir();
 local $ENV{HOME} = $home;
 mkdir "$home/.public-inbox" or BAIL_OUT $!;
@@ -60,6 +60,38 @@ ok(run_script([qw(-extindex --all), "$home/extindex"]), 'extindex init');
        ok($es->has_threadid, '->has_threadid');
 }
 
+if ('with boost') {
+       xsys([qw(git config publicinbox.v1test.boost), 10],
+               { GIT_CONFIG => $cfg_path });
+       ok(run_script([qw(-extindex --all), "$home/extindex-b"]),
+               'extindex init with boost');
+       my $es = PublicInbox::ExtSearch->new("$home/extindex-b");
+       my $smsg = $es->over->get_art(1);
+       ok($smsg, 'got first article');
+       my $xref3 = $es->over->get_xref3($smsg->{num});
+       my @v1 = grep(/\Av1/, @$xref3);
+       my @v2 = grep(/\Av2/, @$xref3);
+       like($v1[0], qr/\Av1\.example.*?\b\Q$smsg->{blob}\E\b/,
+               'smsg->{blob} respected boost');
+       is(scalar(@$xref3), 2, 'only to entries');
+       undef $es;
+
+       xsys([qw(git config publicinbox.v2test.boost), 20],
+               { GIT_CONFIG => $cfg_path });
+       ok(run_script([qw(-extindex --all --reindex), "$home/extindex-b"]),
+               'extindex --reindex with altered boost');
+
+       $es = PublicInbox::ExtSearch->new("$home/extindex-b");
+       $smsg = $es->over->get_art(1);
+       like($v2[0], qr/\Av2\.example.*?\b\Q$smsg->{blob}\E\b/,
+                       'smsg->{blob} respects boost after reindex');
+
+       xsys([qw(git config --unset publicinbox.v1test.boost)],
+               { GIT_CONFIG => $cfg_path });
+       xsys([qw(git config --unset publicinbox.v2test.boost)],
+               { GIT_CONFIG => $cfg_path });
+}
+
 { # TODO: -extindex should write this to config
        open $fh, '>>', $cfg_path or BAIL_OUT $!;
        print $fh <<EOF or BAIL_OUT $!;
@@ -236,6 +268,7 @@ if ('inject w/o indexing') {
 
 if ('reindex catches missed messages') {
        my $v2ibx = $cfg->lookup_name('v2test');
+       $v2ibx->{-no_fsync} = 1;
        my $im = PublicInbox::InboxWritable->new($v2ibx)->importer(0);
        my $cmt_a = $v2ibx->mm->last_commit_xap($schema_version, 0);
        my $eml = eml_load('t/data/0001.patch');
@@ -296,6 +329,7 @@ if ('reindex catches missed messages') {
 if ('reindex catches content bifurcation') {
        use PublicInbox::MID qw(mids);
        my $v2ibx = $cfg->lookup_name('v2test');
+       $v2ibx->{-no_fsync} = 1;
        my $im = PublicInbox::InboxWritable->new($v2ibx)->importer(0);
        my $eml = eml_load('t/data/message_embed.eml');
        my $cmt_a = $v2ibx->mm->last_commit_xap($schema_version, 0);
@@ -366,4 +400,58 @@ if ('remove v1test and test gc') {
        is(scalar(@it), 1, 'only one inbox left');
 }
 
+if ('dedupe + dry-run') {
+       my @cmd = ('-extindex', "$home/extindex");
+       my $opt = { 2 => \(my $err = '') };
+       ok(run_script([@cmd, '--dedupe'], undef, $opt), '--dedupe');
+       ok(run_script([@cmd, qw(--dedupe --dry-run)], undef, $opt),
+               '--dry-run --dedupe');
+       is $err, '', 'no errors';
+       ok(!run_script([@cmd, qw(--dry-run)], undef, $opt),
+               '--dry-run alone fails');
+}
+
+for my $j (1, 3, 6) {
+       my $o = { 2 => \(my $err = '') };
+       my $d = "$home/extindex-j$j";
+       ok(run_script(['-extindex', "-j$j", '--all', $d], undef, $o),
+               "init with -j$j");
+       my $max = $j - 2;
+       $max = 0 if $max < 0;
+       my @dirs = glob("$d/ei*/?");
+       like($dirs[-1], qr!/ei[0-9]+/$max\z!, '-j works');
+}
+
+SKIP: {
+       my $d = "$home/extindex-j1";
+       my $o = { 2 => \(my $err = '') };
+       ok(run_script([qw(-xcpdb -R4), $d]), 'xcpdb R4');
+       my @dirs = glob("$d/ei*/?");
+       for my $i (0..3) {
+               is(grep(m!/ei[0-9]+/$i\z!, @dirs), 1, "shard [$i] created");
+       }
+       for my $i (4..5) {
+               is(grep(m!/ei[0-9]+/$i\z!, @dirs), 0, "no shard [$i]");
+       }
+
+       ok(run_script([qw(-xcpdb -R2), $d]), 'xcpdb -R2');
+       @dirs = glob("$d/ei*/?");
+       for my $i (0..1) {
+               is(grep(m!/ei[0-9]+/$i\z!, @dirs), 1, "shard [$i] kept");
+       }
+       for my $i (2..3) {
+               is(grep(m!/ei[0-9]+/$i\z!, @dirs), 0, "no shard [$i]");
+       }
+       skip 'xapian-compact missing', 4 unless have_xapian_compact;
+       ok(run_script([qw(-compact), $d], undef, $o), 'compact');
+       # n.b. stderr contains xapian-compact output
+
+       my @d2 = glob("$d/ei*/?");
+       is_deeply(\@d2, \@dirs, 'dirs consistent after compact');
+       ok(run_script([qw(-extindex --dedupe --all), $d]),
+               '--dedupe works after compact');
+       ok(run_script([qw(-extindex --gc), $d], undef, $o),
+               '--gc works after compact');
+}
+
 done_testing;