The cost of opening a Xapian DB (even with shards) isn't high,
so save some FDs and just close it. We hit Xapian far less than
over.sqlite3 and we discard the MSet ASAP even when streaming
large responses.
This simplifies our code a bit and hopefully helps reduce
fragmentation by increasing mortality of late allocations.
use Carp qw(croak);
# returns true if further checking is required
use Carp qw(croak);
# returns true if further checking is required
-sub cleanup_shards { $_[0]->{search} ? $_[0]->{search}->cleanup_shards : undef }
-
sub check_inodes ($) {
for (qw(over mm)) { $_[0]->{$_}->check_inodes if $_[0]->{$_} }
}
sub check_inodes ($) {
for (qw(over mm)) { $_[0]->{$_}->check_inodes if $_[0]->{$_} }
}
delete(@$ibx{qw(over mm description cloneurl
-imap_url -nntp_url)});
}
delete(@$ibx{qw(over mm description cloneurl
-imap_url -nntp_url)});
}
- $ibx->cleanup_shards and $live = 1;
+ my $srch = $ibx->{search} // $ibx;
+ delete @$srch{qw(xdb qp)};
for my $git (@{$ibx->{-repo_objs} // []}) {
$live = 1 if $git->cleanup(1);
}
for my $git (@{$ibx->{-repo_objs} // []}) {
$live = 1 if $git->cleanup(1);
}
+sub mm_file {
+ my ($self) = @_;
+ my $d = $self->{inboxdir};
+ ($self->version >= 2 ? $d : "$d/public-inbox").'/msgmap.sqlite3';
+}
+
sub mm {
my ($self, $req) = @_;
$self->{mm} //= eval {
require PublicInbox::Msgmap;
_cleanup_later($self);
sub mm {
my ($self, $req) = @_;
$self->{mm} //= eval {
require PublicInbox::Msgmap;
_cleanup_later($self);
- my $dir = $self->{inboxdir};
- if ($self->version >= 2) {
- PublicInbox::Msgmap->new_file("$dir/msgmap.sqlite3");
- } else {
- PublicInbox::Msgmap->new($dir);
- }
+ PublicInbox::Msgmap->new_file(mm_file($self));
} // ($req ? croak("E: $@") : undef);
}
} // ($req ? croak("E: $@") : undef);
}
-# returns true if a future rescan is desired
-sub cleanup_shards {
- my ($self) = @_;
- return unless exists($self->{xdb});
- my $xpfx = $self->{xpfx};
- return reopen($self) if $xpfx =~ m!/xapian[0-9]+\z!; # true
- opendir(my $dh, $xpfx) or return warn("$xpfx gone: $!\n"); # true
- my $nr = grep(/\A[0-9]+\z/, readdir($dh)) or
- return warn("$xpfx has no shards\n"); # true
- return reopen($self) if $nr == ($self->{nshard} // -1);
- delete @$self{qw(xdb qp)};
- undef;
-}
-
sub new {
my ($class, $ibx) = @_;
ref $ibx or die "BUG: expected PublicInbox::Inbox object: $ibx";
sub new {
my ($class, $ibx) = @_;
ref $ibx or die "BUG: expected PublicInbox::Inbox object: $ibx";
ok(ref($es->{xdb}), '{xdb} created');
my $nshards1 = $es->{nshard};
is($nshards1, 1, 'correct shard count');
ok(ref($es->{xdb}), '{xdb} created');
my $nshards1 = $es->{nshard};
is($nshards1, 1, 'correct shard count');
- my $xdb_str = "$es->{xdb}";
- ok($es->cleanup_shards, 'cleanup_shards noop');
- is("$es->{xdb}", $xdb_str, '{xdb} unchanged');
my @ei_dir = glob("$d/ei*/");
chmod 0755, $ei_dir[0] or xbail "chmod: $!";
my @ei_dir = glob("$d/ei*/");
chmod 0755, $ei_dir[0] or xbail "chmod: $!";
my $m = sprintf('%04o', 07777 & (stat($dirs[$i]))[2]);
is($m, $mode, "shard [$i] mode");
}
my $m = sprintf('%04o', 07777 & (stat($dirs[$i]))[2]);
is($m, $mode, "shard [$i] mode");
}
- is($es->cleanup_shards, undef, 'cleanup_shards cleaned');
- ok(!defined($es->{xdb}), 'old {xdb} gone');
- is($es->cleanup_shards, undef, 'cleanup_shards clean idempotent');
+ delete @$es{qw(xdb qp)};
is($es->mset('z:0..')->size, $nresult0, 'new shards, same results');
is($es->mset('z:0..')->size, $nresult0, 'new shards, same results');
- ok($es->cleanup_shards, 'cleanup_shards true after open');
for my $i (4..5) {
is(grep(m!/ei[0-9]+/$i\z!, @dirs), 0, "no shard [$i]");
for my $i (4..5) {
is(grep(m!/ei[0-9]+/$i\z!, @dirs), 0, "no shard [$i]");