nntp: reduce timers for weakening

Danga::Socket timers are not cheap, so avoid creating up
to 3 timers per-newsgroup by batching resource weakening.
This lets us reduce resource consumption for scheduing
additional resource consumption reduction :)

diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm
index c715e8f7bfcae11d4f5090fe646a2fc093316f20..e143d15ce68fab4e526c0644aa83d475a4713491 100644 (file)
--- a/lib/PublicInbox/NNTP.pm
+++ b/lib/PublicInbox/NNTP.pm
@@ -38,6 +38,8 @@ my %DISABLED; # = map { $_ => 1 } qw(xover list_overview_fmt newnews xhdr);
 my $EXPMAP; # fd -> [ idle_time, $self ]
 my $EXPTIMER;
 our $EXPTIME = 180; # 3 minutes
+my $WEAKEN = {}; # string(nntpd) -> nntpd
+my $WEAKTIMER;
 
 sub update_idle_time ($) {
        my ($self) = @_;
@@ -46,6 +48,16 @@ sub update_idle_time ($) {
        defined $tmp and $EXPMAP->{$tmp} = [ now(), $self ];
 }
 
+# reduce FD pressure by closing some "git cat-file --batch" processes
+# and unused FDs for msgmap and Xapian indices
+sub weaken_groups () {
+       $WEAKTIMER = undef;
+       foreach my $nntpd (values %$WEAKEN) {
+               $_->weaken_all foreach (@{$nntpd->{grouplist}});
+       }
+       $WEAKEN = {};
+}
+
 sub expire_old () {
        my $now = now();
        my $exp = $EXPTIME;
@@ -69,11 +81,15 @@ sub expire_old () {
                $next -= $now;
                $next = 0 if $next < 0;
                $EXPTIMER = Danga::Socket->AddTimer($next, *expire_old);
+               weaken_groups();
        } else {
                $EXPTIMER = undef;
-               # noop to kick outselves out of the loop so descriptors
+               # noop to kick outselves out of the loop ASAP so descriptors
                # really get closed
                Danga::Socket->AddTimer(0, sub {});
+
+               # grace period for reaping resources
+               $WEAKTIMER ||= Danga::Socket->AddTimer(30, *weaken_groups);
        }
 }
 
@@ -87,6 +103,7 @@ sub new ($$$) {
        $self->{rbuf} = '';
        $self->watch_read(1);
        update_idle_time($self);
+       $WEAKEN->{"$nntpd"} = $nntpd;
        $EXPTIMER ||= Danga::Socket->AddTimer($EXPTIME, *expire_old);
        $self;
 }

diff --git a/lib/PublicInbox/NewsGroup.pm b/lib/PublicInbox/NewsGroup.pm
index b20180e6667aceebd0a1feb516118f38603fa69f..adac919fe0b2e15143528d113b2d122f25266a5a 100644 (file)
--- a/lib/PublicInbox/NewsGroup.pm
+++ b/lib/PublicInbox/NewsGroup.pm
@@ -24,20 +24,14 @@ sub new {
        $self;
 }
 
-sub defer_weaken {
-       my ($self, $field) = @_;
-       Danga::Socket->AddTimer(30, sub { weaken($self->{$field}) });
+sub weaken_all {
+       my ($self) = @_;
+       weaken($self->{$_}) foreach qw(gcf mm search);
 }
 
 sub gcf {
        my ($self) = @_;
-       $self->{gcf} ||= eval {
-               my $gcf = PublicInbox::Git->new($self->{git_dir});
-
-               # git repos may be repacked and old packs unlinked
-               defer_weaken($self, 'gcf');
-               $gcf;
-       };
+       $self->{gcf} ||= eval { PublicInbox::Git->new($self->{git_dir}) };
 }
 
 sub usable {
@@ -50,24 +44,12 @@ sub usable {
 
 sub mm {
        my ($self) = @_;
-       $self->{mm} ||= eval {
-               my $mm = PublicInbox::Msgmap->new($self->{git_dir});
-
-               # may be needed if we run low on handles
-               defer_weaken($self, 'mm');
-               $mm;
-       };
+       $self->{mm} ||= eval { PublicInbox::Msgmap->new($self->{git_dir}) };
 }
 
 sub search {
        my ($self) = @_;
-       $self->{search} ||= eval {
-               my $search = PublicInbox::Search->new($self->{git_dir});
-
-               # may be needed if we run low on handles
-               defer_weaken($self, 'search');
-               $search;
-       };
+       $self->{search} ||= eval { PublicInbox::Search->new($self->{git_dir}) };
 }
 
 sub description {