lei q: improve early aborts w/ remote externals

We must issue LeiStore->done if a client disconnects
while we're streaming from a remote external.  This
can happen via SIGPIPE, or if a client process is
interrupted by any other means.

diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index f5e42869e0103c711fddd03cf001b6092ff27635..834e399f1d7b62ffac68db25ca7a69c5d3ddf0ea 100644 (file)
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -970,6 +970,9 @@ sub dclose {
                }
        }
        close(delete $self->{1}) if $self->{1}; # may reap_compress
+       if (my $sto = delete $self->{sto}) {
+               $sto->ipc_do('done');
+       }
        $self->close if $self->{sock}; # PublicInbox::DS::close
 }
 

diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index c2c98030074282e0329767069d41dcd340102054..23cecd53cec816710b8771ae378d3210778926db 100644 (file)
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -18,7 +18,8 @@ sub import_done_wait { # dwaitpid callback
        my ($arg, $pid) = @_;
        my ($imp, $lei) = @$arg;
        $lei->child_error($?, 'non-fatal errors during import') if $?;
-       my $ign = $lei->{sto}->ipc_do('done'); # PublicInbox::LeiStore::done
+       my $sto = delete $lei->{sto};
+       my $wait = $sto->ipc_do('done') if $sto; # PublicInbox::LeiStore::done
        $lei->dclose;
 }
 

diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 9a6457d723fb96642421277aa99166d12cb000ef..d4607e16a331c72b270bcf90598cc2f2067dea2b 100644 (file)
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -349,7 +349,7 @@ Error closing $lei->{ovv}->{dst}: $!
 
 sub do_post_augment {
        my ($lei) = @_;
-       my $l2m = $lei->{l2m} or die 'BUG: unexpected do_post_augment';
+       my $l2m = $lei->{l2m} or return; # client disconnected
        my $err;
        eval { $l2m->post_augment($lei) };
        $err = $@;
@@ -368,7 +368,7 @@ sub do_post_augment {
 
 sub incr_post_augment { # called whenever an l2m shard finishes augment
        my ($lei) = @_;
-       my $l2m = $lei->{l2m} or die 'BUG: unexpected incr_post_augment';
+       my $l2m = $lei->{l2m} or return; # client disconnected
        return if ++$lei->{nr_post_augment} != $l2m->{-wq_nr_workers};
        do_post_augment($lei);
 }

diff --git a/t/lei-externals.t b/t/lei-externals.t
index d422a9d1cad73e4d383b142febff0535d1022a38..b78b5580000e5dfe7b5738cb26fe5774d47ebeed 100644 (file)
--- a/t/lei-externals.t
+++ b/t/lei-externals.t
@@ -6,7 +6,8 @@ use Fcntl qw(SEEK_SET);
 use PublicInbox::Spawn qw(which);
 use PublicInbox::OnDestroy;
 require_git 2.6;
-require_mods(qw(DBD::SQLite Search::Xapian));
+require_mods(qw(json DBD::SQLite Search::Xapian));
+use POSIX qw(WTERMSIG WIFSIGNALED SIGPIPE);
 
 my @onions = qw(http://hjrcffqmbrq6wope.onion/meta/
        http://czquwvybam4bgbro.onion/meta/
@@ -15,19 +16,55 @@ my @onions = qw(http://hjrcffqmbrq6wope.onion/meta/
 my $test_external_remote = sub {
        my ($url, $k) = @_;
 SKIP: {
-       my $nr = 5;
-       skip "$k unset", $nr if !$url;
-       which('curl') or skip 'no curl', $nr;
-       which('torsocks') or skip 'no torsocks', $nr if $url =~ m!\.onion/!;
+       skip "$k unset", 1 if !$url;
+       state $curl = which('curl');
+       $curl or skip 'no curl', 1;
+       which('torsocks') or skip 'no torsocks', 1 if $url =~ m!\.onion/!;
        my $mid = '20140421094015.GA8962@dcvr.yhbt.net';
        my @cmd = ('q', '--only', $url, '-q', "m:$mid");
        lei_ok(@cmd, \"query $url");
        is($lei_err, '', "no errors on $url");
        my $res = json_utf8->decode($lei_out);
-       is($res->[0]->{'m'}, "<$mid>", "got expected mid from $url");
+       is($res->[0]->{'m'}, "<$mid>", "got expected mid from $url") or
+               skip 'further remote tests', 1;
        lei_ok(@cmd, 'd:..20101002', \'no results, no error');
        is($lei_err, '', 'no output on 404, matching local FS behavior');
        is($lei_out, "[null]\n", 'got null results');
+       my ($pid_before, $pid_after);
+       if (-d $ENV{XDG_RUNTIME_DIR} && -w _) {
+               lei_ok 'daemon-pid';
+               chomp($pid_before = $lei_out);
+               ok($pid_before, 'daemon is live');
+       }
+       for my $out ([], [qw(-f mboxcl2)]) {
+               pipe(my ($r, $w)) or BAIL_OUT $!;
+               open my $err, '+>', undef or BAIL_OUT $!;
+               my $opt = { run_mode => 0, 1 => $w, 2 => $err };
+               my $cmd = [qw(lei q -qt), @$out, 'bytes:1..'];
+               my $tp = start_script($cmd, undef, $opt);
+               close $w;
+               sysread($r, my $buf, 1);
+               close $r; # trigger SIGPIPE
+               $tp->join;
+               ok(WIFSIGNALED($?), "signaled @$out");
+               is(WTERMSIG($?), SIGPIPE, "got SIGPIPE @$out");
+               seek($err, 0, 0);
+               my @err = grep(!m{mkdir .*sun_path\b}, <$err>);
+               is_deeply(\@err, [], "no errors @$out");
+       }
+       if (-d $ENV{XDG_RUNTIME_DIR} && -w _) {
+               lei_ok 'daemon-pid';
+               chomp(my $pid_after = $lei_out);
+               is($pid_after, $pid_before, 'pid unchanged') or
+                       skip 'daemon died', 1;
+               lei_ok 'daemon-kill';
+               my $alive = 1;
+               for (1..100) {
+                       $alive = kill(0, $pid_after) or last;
+                       tick();
+               }
+               ok(!$alive, 'daemon-kill worked');
+       }
 } # /SKIP
 }; # /sub