daemon: unconditionally close Xapian shards on cleanup

The cost of opening a Xapian DB (even with shards) isn't high,
so save some FDs and just close it.  We hit Xapian far less than
over.sqlite3 and we discard the MSet ASAP even when streaming
large responses.

This simplifies our code a bit and hopefully helps reduce
fragmentation by increasing mortality of late allocations.

diff --git a/lib/PublicInbox/Inbox.pm b/lib/PublicInbox/Inbox.pm
index 724df50a7134725601cd1c2ce83d9a9f41805c77..61d153bfab289c4885bfe828163ab48891467754 100644 (file)
--- a/lib/PublicInbox/Inbox.pm
+++ b/lib/PublicInbox/Inbox.pm
@@ -11,8 +11,6 @@ use List::Util qw(max);
 use Carp qw(croak);
 
 # returns true if further checking is required
-sub cleanup_shards { $_[0]->{search} ? $_[0]->{search}->cleanup_shards : undef }
-
 sub check_inodes ($) {
        for (qw(over mm)) { $_[0]->{$_}->check_inodes if $_[0]->{$_} }
 }
@@ -31,7 +29,8 @@ sub do_cleanup {
                delete(@$ibx{qw(over mm description cloneurl
                                -imap_url -nntp_url)});
        }
-       $ibx->cleanup_shards and $live = 1;
+       my $srch = $ibx->{search} // $ibx;
+       delete @$srch{qw(xdb qp)};
        for my $git (@{$ibx->{-repo_objs} // []}) {
                $live = 1 if $git->cleanup(1);
        }
@@ -138,17 +137,18 @@ sub max_git_epoch {
        $cur;
 }
 
+sub mm_file {
+       my ($self) = @_;
+       my $d = $self->{inboxdir};
+       ($self->version >= 2 ? $d : "$d/public-inbox").'/msgmap.sqlite3';
+}
+
 sub mm {
        my ($self, $req) = @_;
        $self->{mm} //= eval {
                require PublicInbox::Msgmap;
                _cleanup_later($self);
-               my $dir = $self->{inboxdir};
-               if ($self->version >= 2) {
-                       PublicInbox::Msgmap->new_file("$dir/msgmap.sqlite3");
-               } else {
-                       PublicInbox::Msgmap->new($dir);
-               }
+               PublicInbox::Msgmap->new_file(mm_file($self));
        } // ($req ? croak("E: $@") : undef);
 }
 

diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm
index dd6d3710d03d48592d835f302b02eddce82f52df..f0e7ed0c2665d43308682986d6612ae1d1696000 100644 (file)
--- a/lib/PublicInbox/Search.pm
+++ b/lib/PublicInbox/Search.pm
@@ -243,20 +243,6 @@ sub xdb ($) {
        };
 }
 
-# returns true if a future rescan is desired
-sub cleanup_shards {
-       my ($self) = @_;
-       return unless exists($self->{xdb});
-       my $xpfx = $self->{xpfx};
-       return reopen($self) if $xpfx =~ m!/xapian[0-9]+\z!; # true
-       opendir(my $dh, $xpfx) or return warn("$xpfx gone: $!\n"); # true
-       my $nr = grep(/\A[0-9]+\z/, readdir($dh)) or
-               return warn("$xpfx has no shards\n"); # true
-       return reopen($self) if $nr == ($self->{nshard} // -1);
-       delete @$self{qw(xdb qp)};
-       undef;
-}
-
 sub new {
        my ($class, $ibx) = @_;
        ref $ibx or die "BUG: expected PublicInbox::Inbox object: $ibx";

diff --git a/t/extsearch.t b/t/extsearch.t
index 8190de17cf16d3fdd9ef28e7cde0f14c2a895904..dfc190e2d69e8a3d2628361a62db45d1102c86f2 100644 (file)
--- a/t/extsearch.t
+++ b/t/extsearch.t
@@ -447,9 +447,6 @@ SKIP: {
        ok(ref($es->{xdb}), '{xdb} created');
        my $nshards1 = $es->{nshard};
        is($nshards1, 1, 'correct shard count');
-       my $xdb_str = "$es->{xdb}";
-       ok($es->cleanup_shards, 'cleanup_shards noop');
-       is("$es->{xdb}", $xdb_str, '{xdb} unchanged');
 
        my @ei_dir = glob("$d/ei*/");
        chmod 0755, $ei_dir[0] or xbail "chmod: $!";
@@ -463,11 +460,8 @@ SKIP: {
                my $m = sprintf('%04o', 07777 & (stat($dirs[$i]))[2]);
                is($m, $mode, "shard [$i] mode");
        }
-       is($es->cleanup_shards, undef, 'cleanup_shards cleaned');
-       ok(!defined($es->{xdb}), 'old {xdb} gone');
-       is($es->cleanup_shards, undef, 'cleanup_shards clean idempotent');
+       delete @$es{qw(xdb qp)};
        is($es->mset('z:0..')->size, $nresult0, 'new shards, same results');
-       ok($es->cleanup_shards, 'cleanup_shards true after open');
 
        for my $i (4..5) {
                is(grep(m!/ei[0-9]+/$i\z!, @dirs), 0, "no shard [$i]");