use List::Util qw(max);
use Carp qw(croak);
-# Long-running "git-cat-file --batch" processes won't notice
-# unlinked packs, so we need to restart those processes occasionally.
-# Xapian and SQLite file handles are mostly stable, but sometimes an
-# admin will attempt to replace them atomically after compact/vacuum
-# and we need to be prepared for that.
-my $cleanup_timer;
-my $CLEANUP = {}; # string(inbox) -> inbox
-
-sub git_cleanup ($) {
- my ($self) = @_;
- my $git = $self->{git} // return undef;
- # normal inboxes have low startup cost and there may be many, so
- # keep process+pipe counts in check. ExtSearch may have high startup
- # cost (e.g. ->ALL) and but likely one per-daemon, so cleanup only
- # if there's unlinked files
- my $live = $self->isa(__PACKAGE__) ? $git->cleanup
- : $git->cleanup_if_unlinked;
- delete($self->{git}) unless $live;
- $live;
+# returns true if further checking is required
+sub check_inodes ($) {
+ for (qw(over mm)) { $_[0]->{$_}->check_inodes if $_[0]->{$_} }
}
-# returns true if further checking is required
-sub cleanup_shards { $_[0]->{search} ? $_[0]->{search}->cleanup_shards : undef }
-
-sub cleanup_task () {
- $cleanup_timer = undef;
- my $next = {};
- for my $ibx (values %$CLEANUP) {
- my $again = git_cleanup($ibx);
- $ibx->cleanup_shards and $again = 1;
- for my $git (@{$ibx->{-repo_objs}}) {
- $again = 1 if $git->cleanup;
- }
+sub do_cleanup {
+ my ($ibx) = @_;
+ my $live;
+ if (defined $ibx->{git}) {
+ $live = $ibx->isa(__PACKAGE__) ? $ibx->{git}->cleanup(1)
+ : $ibx->{git}->cleanup_if_unlinked;
+ delete($ibx->{git}) unless $live;
+ }
+ if ($live) {
check_inodes($ibx);
- $next->{"$ibx"} = $ibx if $again;
+ } else {
+ delete(@$ibx{qw(over mm description cloneurl
+ -imap_url -nntp_url)});
+ }
+ my $srch = $ibx->{search} // $ibx;
+ delete @$srch{qw(xdb qp)};
+ for my $git (@{$ibx->{-repo_objs} // []}) {
+ $live = 1 if $git->cleanup(1);
}
- $CLEANUP = $next;
- $cleanup_timer //= PublicInbox::DS::later(\&cleanup_task);
+ PublicInbox::DS::add_uniq_timer($ibx+0, 5, \&do_cleanup, $ibx) if $live;
}
sub _cleanup_later ($) {
# no need to require DS, here, if it were enabled another
# module would've require'd it, already
- if (eval { PublicInbox::DS::in_loop() }) {
- $cleanup_timer //= PublicInbox::DS::later(\&cleanup_task);
- $CLEANUP->{"$_[0]"} = $_[0]; # $self
- }
+ eval { PublicInbox::DS::in_loop() } and
+ PublicInbox::DS::add_uniq_timer($_[0]+0, 30, \&do_cleanup, @_)
}
sub _set_limiter ($$$) {
my $cur = $self->{-max_git_epoch};
my $changed;
if (!defined($cur) || ($changed = git($self)->alternates_changed)) {
- git_cleanup($self) if $changed;
+ $self->{git}->cleanup if $changed;
my $gits = "$self->{inboxdir}/git";
if (opendir my $dh, $gits) {
my $max = max(map {
$cur;
}
+sub mm_file {
+ my ($self) = @_;
+ my $d = $self->{inboxdir};
+ ($self->version >= 2 ? $d : "$d/public-inbox").'/msgmap.sqlite3';
+}
+
sub mm {
my ($self, $req) = @_;
$self->{mm} //= eval {
require PublicInbox::Msgmap;
- my $dir = $self->{inboxdir};
- if ($self->version >= 2) {
- PublicInbox::Msgmap->new_file("$dir/msgmap.sqlite3");
- } else {
- PublicInbox::Msgmap->new($dir);
- }
+ _cleanup_later($self);
+ PublicInbox::Msgmap->new_file(mm_file($self));
} // ($req ? croak("E: $@") : undef);
}
require PublicInbox::Search;
PublicInbox::Search->new($self);
};
+ _cleanup_later($self);
my $over = PublicInbox::Over->new("$srch->{xpfx}/over.sqlite3");
$over->dbh; # may fail
$self->{over} = $over;
return $url .= $self->{name} . '/';
}
# called from a non-PSGI environment (e.g. NNTP/POP3):
- $self->{-base_url} ||= do {
- my $url = $self->{url} // return undef;
- $url = $url->[0] // return undef;
- # expand protocol-relative URLs to HTTPS if we're
- # not inside a web server
- $url = "https:$url" if $url =~ m!\A//!;
- $url .= '/' if $url !~ m!/\z!;
- $url;
- };
+ my $url = $self->{url} // return undef;
+ $url = $url->[0] // return undef;
+ # expand protocol-relative URLs to HTTPS if we're
+ # not inside a web server
+ substr($url, 0, 0, 'https:') if substr($url, 0, 2) eq '//';
+ $url .= '/' if substr($url, -1, 1) ne '/';
+ $url;
}
sub _x_url ($$$) {
# (pathname is NOT public, but prefix is used for Xapian queries)
sub altid_map ($) {
my ($self) = @_;
- $self->{-altid_map} //= eval {
+ eval {
require PublicInbox::AltId;
my $altid = $self->{altid} or return {};
my %h = map {;
delete $self->{unlock_subs}->{$ident};
}
-sub check_inodes ($) {
- my ($self) = @_;
- for (qw(over mm)) { # TODO: search
- $self->{$_}->check_inodes if $self->{$_};
- }
-}
-
# called by inotify
sub on_unlock {
my ($self) = @_;