ipc: add wq_broadcast

We'll give workqueues a broadcast mechanism to ensure all
workers see a certain message.  We'll also tag each worker
with {-wq_worker_nr} in preparation for work distribution.

This is intended to avoid extra connection and fork() costs
from LeiAuth in a future commit.

diff --git a/lib/PublicInbox/IPC.pm b/lib/PublicInbox/IPC.pm
index efac4c4dd9a145372a96d2de0e37d6b7ff2e1252..2aeb64623cb0e6dd80895f1c36c7a50ebdce6939 100644 (file)
--- a/lib/PublicInbox/IPC.pm
+++ b/lib/PublicInbox/IPC.pm
@@ -240,8 +240,9 @@ sub recv_and_run {
 }
 
 sub wq_worker_loop ($) {
-       my ($self) = @_;
+       my ($self, $bcast_a) = @_;
        my $wqw = PublicInbox::WQWorker->new($self);
+       PublicInbox::WQWorker->new($self, '-wq_bcast2');
        PublicInbox::DS->SetPostLoopCallback(sub { $wqw->{sock} });
        PublicInbox::DS->EventLoop;
        PublicInbox::DS->Reset;
@@ -252,6 +253,20 @@ sub do_sock_stream { # via wq_io_do, for big requests
        recv_and_run($self, delete $self->{0}, $len, 1);
 }
 
+sub wq_broadcast {
+       my ($self, $sub, @args) = @_;
+       if (my $wkr = $self->{-wq_workers}) {
+               for my $bcast1 (values %$wkr) {
+                       my $buf = ipc_freeze([$sub, @args]);
+                       send($bcast1, $buf, MSG_EOR) // croak "send: $!";
+                       # XXX shouldn't have to deal with EMSGSIZE here...
+               }
+       } else {
+               eval { $self->$sub(@args) };
+               warn "wq_broadcast: $@" if $@;
+       }
+}
+
 sub wq_io_do { # always async
        my ($self, $sub, $ios, @args) = @_;
        if (my $s1 = $self->{-wq_s1}) { # run in worker
@@ -284,15 +299,21 @@ sub wq_io_do { # always async
 
 sub _wq_worker_start ($$$) {
        my ($self, $oldset, $fields) = @_;
+       my ($bcast1, $bcast2);
+       socketpair($bcast1, $bcast2, AF_UNIX, $SEQPACKET, 0) or
+                                               die "socketpair: $!";
        my $seed = rand(0xffffffff);
        my $pid = fork // die "fork: $!";
        if ($pid == 0) {
                srand($seed);
+               undef $bcast1;
                eval { PublicInbox::DS->Reset };
-               delete @$self{qw(-wq_s1 -wq_workers -wq_ppid)};
+               delete @$self{qw(-wq_s1 -wq_ppid)};
+               $self->{-wq_worker_nr} =
+                               keys %{delete($self->{-wq_workers}) // {}};
                $SIG{$_} = 'IGNORE' for (qw(PIPE));
                $SIG{$_} = 'DEFAULT' for (qw(TTOU TTIN TERM QUIT INT CHLD));
-               local $0 = $self->{-wq_ident};
+               local $0 = "$self->{-wq_ident} $self->{-wq_worker_nr}";
                # ensure we properly exit even if warn() dies:
                my $end = PublicInbox::OnDestroy->new($$, sub { exit(!!$@) });
                eval {
@@ -301,12 +322,13 @@ sub _wq_worker_start ($$$) {
                        my $on_destroy = $self->ipc_atfork_child;
                        local %SIG = %SIG;
                        PublicInbox::DS::sig_setmask($oldset);
+                       $self->{-wq_bcast2} = $bcast2;
                        wq_worker_loop($self);
                };
                warn "worker $self->{-wq_ident} PID:$$ died: $@" if $@;
                undef $end; # trigger exit
        } else {
-               $self->{-wq_workers}->{$pid} = \undef;
+               $self->{-wq_workers}->{$pid} = $bcast1;
        }
 }
 

diff --git a/lib/PublicInbox/WQWorker.pm b/lib/PublicInbox/WQWorker.pm
index 25a5e4fbf9887d222264ec09fc451080187cc71d..3636321e901eab5b365922331ee819767d8777b8 100644 (file)
--- a/lib/PublicInbox/WQWorker.pm
+++ b/lib/PublicInbox/WQWorker.pm
@@ -11,10 +11,10 @@ use Errno qw(EAGAIN ECONNRESET);
 use IO::Handle (); # blocking
 
 sub new {
-       my (undef, $wq) = @_;
-       my $s2 = $wq->{-wq_s2} // die 'BUG: no -wq_s2';
+       my ($cls, $wq, $field) = @_;
+       my $s2 = $wq->{$field // '-wq_s2'} // die "BUG: no {$field}";
        $s2->blocking(0);
-       my $self = bless { sock => $s2, wq => $wq }, __PACKAGE__;
+       my $self = bless { sock => $s2, wq => $wq }, $cls;
        $self->SUPER::new($s2, EPOLLEXCLUSIVE|EPOLLIN|EPOLLET);
        $self;
 }
@@ -27,7 +27,7 @@ sub event_step {
        } while ($n);
        return if !defined($n) && $! == EAGAIN; # likely
        warn "wq worker error: $!\n" if !defined($n) && $! != ECONNRESET;
-       $self->{wq}->wq_atexit_child;
+       $self->{wq}->wq_atexit_child if $self->{sock} == $self->{wq}->{-wq_s2};
        $self->close; # PublicInbox::DS::close
 }
 

diff --git a/t/ipc.t b/t/ipc.t
index 345024bd9038e3bcab03e9cce205305b4211226c..ca88eb59d4ed8fe4dc16fcd2d24ea7c973e060c6 100644 (file)
--- a/t/ipc.t
+++ b/t/ipc.t
@@ -9,6 +9,7 @@ use Fcntl qw(SEEK_SET);
 use Digest::SHA qw(sha1_hex);
 require_mods(qw(Storable||Sereal));
 require_ok 'PublicInbox::IPC';
+my ($tmpdir, $for_destroy) = tmpdir();
 state $once = eval <<'';
 package PublicInbox::IPC;
 use strict;
@@ -31,6 +32,12 @@ sub test_sha {
        print { $self->{1} } sha1_hex($buf), "\n";
        $self->{1}->flush;
 }
+sub test_append_pid {
+       my ($self, $file) = @_;
+       open my $fh, '>>', $file or die "open: $!";
+       $fh->autoflush(1);
+       print $fh "$$\n" or die "print: $!";
+}
 1;
 
 my $ipc = bless {}, 'PublicInbox::IPC';
@@ -83,11 +90,8 @@ $test->('local');
        defined($pid) or BAIL_OUT 'no spawn, no test';
        is($ipc->ipc_do('test_pid'), $pid, 'worker pid returned');
        $test->('worker');
-       {
-               my ($tmp, $for_destroy) = tmpdir();
-               $ipc->ipc_lock_init("$tmp/lock");
-               is($ipc->ipc_do('test_pid'), $pid, 'worker pid returned');
-       }
+       $ipc->ipc_lock_init("$tmpdir/lock");
+       is($ipc->ipc_do('test_pid'), $pid, 'worker pid returned');
        $ipc->ipc_worker_stop;
        ok(!kill(0, $pid) && $!{ESRCH}, 'worker stopped');
 }
@@ -167,18 +171,21 @@ SKIP: {
        $SIG{__WARN__} = 'DEFAULT';
        is($ipc->wq_workers_start('wq', 1), $$, 'workers started again');
        is($ipc->wq_workers, 1, '1 worker started');
-       SKIP: {
-               $ipc->WQ_MAX_WORKERS > 1 or
-                       skip 'Inline::C or Socket::MsgHdr not available', 4;
-               $ipc->wq_worker_incr;
-               is($ipc->wq_workers, 2, 'worker count bumped');
-               $ipc->wq_worker_decr;
-               $ipc->wq_worker_decr_wait(10);
-               is($ipc->wq_workers, 1, 'worker count lowered');
-               is($ipc->wq_workers(2), 2, 'worker count set');
-               is($ipc->wq_workers, 2, 'worker count stayed set');
-       }
+
+       $ipc->wq_worker_incr;
+       is($ipc->wq_workers, 2, 'worker count bumped');
+       $ipc->wq_worker_decr;
+       $ipc->wq_worker_decr_wait(10);
+       is($ipc->wq_workers, 1, 'worker count lowered');
+       is($ipc->wq_workers(2), 2, 'worker count set');
+       is($ipc->wq_workers, 2, 'worker count stayed set');
+
+       $ipc->wq_broadcast('test_append_pid', "$tmpdir/append_pid");
        $ipc->wq_close;
+       open my $fh, '<', "$tmpdir/append_pid" or BAIL_OUT "open: $!";
+       chomp(my @pids = <$fh>);
+       my %pids = map { $_ => 1 } grep(/\A[0-9]+\z/, @pids);
+       is(scalar keys %pids, 2, 'broadcast hit both PIDs');
        is($ipc->wq_workers, undef, 'workers undef after close');
 }