From 2a3c8d7a2c40ad9424db10d68470d8bb120bddf9 Mon Sep 17 00:00:00 2001
From: Eric Wong <e@80x24.org>
Date: Tue, 12 Oct 2021 11:47:03 +0000
Subject: [PATCH] daemon: unconditionally close Xapian shards on cleanup

The cost of opening a Xapian DB (even with shards) isn't high,
so save some FDs and just close it.  We hit Xapian far less than
over.sqlite3 and we discard the MSet ASAP even when streaming
large responses.

This simplifies our code a bit and hopefully helps reduce
fragmentation by increasing mortality of late allocations.
---
 lib/PublicInbox/Inbox.pm  | 18 +++++++++---------
 lib/PublicInbox/Search.pm | 14 --------------
 t/extsearch.t             |  8 +-------
 3 files changed, 10 insertions(+), 30 deletions(-)

diff --git a/lib/PublicInbox/Inbox.pm b/lib/PublicInbox/Inbox.pm
index 724df50a..61d153bf 100644
--- a/lib/PublicInbox/Inbox.pm
+++ b/lib/PublicInbox/Inbox.pm
@@ -11,8 +11,6 @@ use List::Util qw(max);
 use Carp qw(croak);
 
 # returns true if further checking is required
-sub cleanup_shards { $_[0]->{search} ? $_[0]->{search}->cleanup_shards : undef }
-
 sub check_inodes ($) {
 	for (qw(over mm)) { $_[0]->{$_}->check_inodes if $_[0]->{$_} }
 }
@@ -31,7 +29,8 @@ sub do_cleanup {
 		delete(@$ibx{qw(over mm description cloneurl
 				-imap_url -nntp_url)});
 	}
-	$ibx->cleanup_shards and $live = 1;
+	my $srch = $ibx->{search} // $ibx;
+	delete @$srch{qw(xdb qp)};
 	for my $git (@{$ibx->{-repo_objs} // []}) {
 		$live = 1 if $git->cleanup(1);
 	}
@@ -138,17 +137,18 @@ sub max_git_epoch {
 	$cur;
 }
 
+sub mm_file {
+	my ($self) = @_;
+	my $d = $self->{inboxdir};
+	($self->version >= 2 ? $d : "$d/public-inbox").'/msgmap.sqlite3';
+}
+
 sub mm {
 	my ($self, $req) = @_;
 	$self->{mm} //= eval {
 		require PublicInbox::Msgmap;
 		_cleanup_later($self);
-		my $dir = $self->{inboxdir};
-		if ($self->version >= 2) {
-			PublicInbox::Msgmap->new_file("$dir/msgmap.sqlite3");
-		} else {
-			PublicInbox::Msgmap->new($dir);
-		}
+		PublicInbox::Msgmap->new_file(mm_file($self));
 	} // ($req ? croak("E: $@") : undef);
 }
 
diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm
index dd6d3710..f0e7ed0c 100644
--- a/lib/PublicInbox/Search.pm
+++ b/lib/PublicInbox/Search.pm
@@ -243,20 +243,6 @@ sub xdb ($) {
 	};
 }
 
-# returns true if a future rescan is desired
-sub cleanup_shards {
-	my ($self) = @_;
-	return unless exists($self->{xdb});
-	my $xpfx = $self->{xpfx};
-	return reopen($self) if $xpfx =~ m!/xapian[0-9]+\z!; # true
-	opendir(my $dh, $xpfx) or return warn("$xpfx gone: $!\n"); # true
-	my $nr = grep(/\A[0-9]+\z/, readdir($dh)) or
-		return warn("$xpfx has no shards\n"); # true
-	return reopen($self) if $nr == ($self->{nshard} // -1);
-	delete @$self{qw(xdb qp)};
-	undef;
-}
-
 sub new {
 	my ($class, $ibx) = @_;
 	ref $ibx or die "BUG: expected PublicInbox::Inbox object: $ibx";
diff --git a/t/extsearch.t b/t/extsearch.t
index 8190de17..dfc190e2 100644
--- a/t/extsearch.t
+++ b/t/extsearch.t
@@ -447,9 +447,6 @@ SKIP: {
 	ok(ref($es->{xdb}), '{xdb} created');
 	my $nshards1 = $es->{nshard};
 	is($nshards1, 1, 'correct shard count');
-	my $xdb_str = "$es->{xdb}";
-	ok($es->cleanup_shards, 'cleanup_shards noop');
-	is("$es->{xdb}", $xdb_str, '{xdb} unchanged');
 
 	my @ei_dir = glob("$d/ei*/");
 	chmod 0755, $ei_dir[0] or xbail "chmod: $!";
@@ -463,11 +460,8 @@ SKIP: {
 		my $m = sprintf('%04o', 07777 & (stat($dirs[$i]))[2]);
 		is($m, $mode, "shard [$i] mode");
 	}
-	is($es->cleanup_shards, undef, 'cleanup_shards cleaned');
-	ok(!defined($es->{xdb}), 'old {xdb} gone');
-	is($es->cleanup_shards, undef, 'cleanup_shards clean idempotent');
+	delete @$es{qw(xdb qp)};
 	is($es->mset('z:0..')->size, $nresult0, 'new shards, same results');
-	ok($es->cleanup_shards, 'cleanup_shards true after open');
 
 	for my $i (4..5) {
 		is(grep(m!/ei[0-9]+/$i\z!, @dirs), 0, "no shard [$i]");
-- 
2.44.0