watch: use EOFpipe to reduce dwaitpid wakeups

[public-inbox.git] / lib / PublicInbox / Watch.pm

diff --git a/lib/PublicInbox/Watch.pm b/lib/PublicInbox/Watch.pm
index 0bb92d0a9055703233b7af5cafe53bdfa0232e2f..17786377ba8127fe7744acc14216f873ec71edf7 100644 (file)
--- a/lib/PublicInbox/Watch.pm
+++ b/lib/PublicInbox/Watch.pm
@@ -14,7 +14,8 @@ use PublicInbox::Sigfd;
 use PublicInbox::DS qw(now);
 use PublicInbox::MID qw(mids);
 use PublicInbox::ContentHash qw(content_hash);
-use POSIX qw(_exit);
+use PublicInbox::EOFpipe;
+use POSIX qw(_exit WNOHANG);
 
 sub compile_watchheaders ($) {
        my ($ibx) = @_;
@@ -268,7 +269,7 @@ sub watch_fs_init ($) {
                delete $self->{done_timer};
                _done_for_now($self);
        };
-       my $cb = sub {
+       my $cb = sub { # called by PublicInbox::DirIdle::event_step
                _try_path($self, $_[0]->fullname);
                $self->{done_timer} //= PublicInbox::DS::requeue($done);
        };
@@ -411,7 +412,7 @@ sub imap_import_msg ($$$$$) {
        if (ref($inboxes)) {
                for my $ibx (@$inboxes) {
                        my $eml = PublicInbox::Eml->new($$raw);
-                       my $x = import_eml($self, $ibx, $eml);
+                       import_eml($self, $ibx, $eml);
                }
        } elsif ($inboxes eq 'watchspam') {
                # we don't remove unseen messages
@@ -566,7 +567,7 @@ sub watch_imap_idle_1 ($$$) {
                        $err = imap_fetch_all($self, $mic, $url);
                        $err //= imap_idle_once($self, $mic, $intvl, $url);
                } else {
-                       $err = "not connected: $!";
+                       $err = "E: not connected: $!";
                }
                if ($err && !$self->{quit}) {
                        warn $err, "\n";
@@ -589,6 +590,7 @@ sub watch_atfork_child ($) {
 sub watch_atfork_parent ($) {
        my ($self) = @_;
        _done_for_now($self);
+       PublicInbox::Sigfd::block_signals();
 }
 
 sub imap_idle_requeue ($) { # DS::add_timer callback
@@ -610,17 +612,30 @@ sub imap_idle_reap { # PublicInbox::DS::dwaitpid callback
                                \&imap_idle_requeue, [ $self, $url_intvl ]);
 }
 
+sub reap { # callback for EOFpipe
+       my ($pid, $cb, $self) = @{$_[0]};
+       my $ret = waitpid($pid, 0);
+       if ($ret == $pid) {
+               $cb->($self, $pid); # poll_fetch_reap || imap_idle_reap
+       } else {
+               warn "W: waitpid($pid) => ", $ret // "($!)", "\n";
+       }
+}
+
 sub imap_idle_fork ($$) {
        my ($self, $url_intvl) = @_;
        my ($url, $intvl) = @$url_intvl;
+       pipe(my ($r, $w)) or die "pipe: $!";
        defined(my $pid = fork) or die "fork: $!";
        if ($pid == 0) {
+               close $r;
                watch_atfork_child($self);
                watch_imap_idle_1($self, $url, $intvl);
+               close $w;
                _exit(0);
        }
        $self->{idle_pids}->{$pid} = $url_intvl;
-       PublicInbox::DS::dwaitpid($pid, \&imap_idle_reap, $self);
+       PublicInbox::EOFpipe->new($r, \&reap, [$pid, \&imap_idle_reap, $self]);
 }
 
 sub event_step {
@@ -628,10 +643,14 @@ sub event_step {
        return if $self->{quit};
        my $idle_todo = $self->{idle_todo};
        if ($idle_todo && @$idle_todo) {
-               watch_atfork_parent($self);
-               while (my $url_intvl = shift(@$idle_todo)) {
-                       imap_idle_fork($self, $url_intvl);
-               }
+               my $oldset = watch_atfork_parent($self);
+               eval {
+                       while (my $url_intvl = shift(@$idle_todo)) {
+                               imap_idle_fork($self, $url_intvl);
+                       }
+               };
+               PublicInbox::Sigfd::sig_setmask($oldset);
+               die $@ if $@;
        }
        goto(&fs_scan_step) if $self->{mdre};
 }
@@ -684,22 +703,27 @@ sub watch_nntp_fetch_all ($$) {
 sub poll_fetch_fork ($) { # DS::add_timer callback
        my ($self, $intvl, $urls) = @{$_[0]};
        return if $self->{quit};
-       watch_atfork_parent($self);
-       defined(my $pid = fork) or die "fork: $!";
-       if ($pid == 0) {
+       pipe(my ($r, $w)) or die "pipe: $!";
+       my $oldset = watch_atfork_parent($self);
+       my $pid = fork;
+       if (defined($pid) && $pid == 0) {
+               close $r;
                watch_atfork_child($self);
                if ($urls->[0] =~ m!\Aimaps?://!i) {
                        watch_imap_fetch_all($self, $urls);
                } else {
                        watch_nntp_fetch_all($self, $urls);
                }
+               close $w;
                _exit(0);
        }
+       PublicInbox::Sigfd::sig_setmask($oldset);
+       die "fork: $!"  unless defined $pid;
        $self->{poll_pids}->{$pid} = [ $intvl, $urls ];
-       PublicInbox::DS::dwaitpid($pid, \&poll_fetch_reap, $self);
+       PublicInbox::EOFpipe->new($r, \&reap, [$pid, \&poll_fetch_reap, $self]);
 }
 
-sub poll_fetch_reap { # PublicInbox::DS::dwaitpid callback
+sub poll_fetch_reap {
        my ($self, $pid) = @_;
        my $intvl_urls = delete $self->{poll_pids}->{$pid} or
                die "BUG: PID=$pid (unknown) reaped: \$?=$?\n";
@@ -977,7 +1001,7 @@ sub watch_nntp_init ($$) {
        }
 }
 
-sub watch {
+sub watch { # main entry point
        my ($self, $sig, $oldset) = @_;
        $self->{oldset} = $oldset;
        $self->{sig} = $sig;
@@ -991,7 +1015,7 @@ sub watch {
        }
        watch_fs_init($self) if $self->{mdre};
        PublicInbox::DS->SetPostLoopCallback(sub { !$self->quit_done });
-       PublicInbox::DS->EventLoop;
+       PublicInbox::DS->EventLoop; # calls ->event_step
        _done_for_now($self);
 }
 
@@ -1076,7 +1100,7 @@ sub content_exists ($$) {
 
 sub _spamcheck_cb {
        my ($sc) = @_;
-       sub {
+       sub { # this gets called by (V2Writable||Import)->add
                my ($mime, $ibx) = @_;
                return if content_exists($ibx, $mime);
                my $tmp = '';