lei_input: support PublicInbox::WWW mboxrd URLs

This gives "lei import", "lei tag", and similar commands
the ability to use URLs recognized by our PSGI frontend
directly.

This is more convenient than an equivalent shell pipeline
since "set -o pipefail" is not portable and errors may be
lost.

diff --git a/MANIFEST b/MANIFEST
index 79d393c5760a325f63336291146191e86f3032d5..ce824fcf3276cfc8792085ee6e385430a4fad94d 100644 (file)
--- a/MANIFEST
+++ b/MANIFEST
@@ -394,6 +394,7 @@ t/kqnotify.t
 t/lei-convert.t
 t/lei-daemon.t
 t/lei-externals.t
+t/lei-import-http.t
 t/lei-import-imap.t
 t/lei-import-maildir.t
 t/lei-import-nntp.t

diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm
index d11d23d4064d374789b8200cab8c23101377e804..785e607d783bf59a52d3bb7d66cd637af8ed1444 100644 (file)
--- a/lib/PublicInbox/LeiInput.pm
+++ b/lib/PublicInbox/LeiInput.pm
@@ -6,6 +6,7 @@ package PublicInbox::LeiInput;
 use strict;
 use v5.10.1;
 use PublicInbox::DS;
+use PublicInbox::Spawn qw(which popen_rd);
 
 # JMAP RFC 8621 4.1.1
 # https://www.iana.org/assignments/imap-jmap-keywords/imap-jmap-keywords.xhtml
@@ -77,6 +78,32 @@ error reading $name: $!
        }
 }
 
+# handles mboxrd endpoints described in Documentation/design_notes.txt
+sub handle_http_input ($$@) {
+       my ($self, $url, @args) = @_;
+       my $lei = $self->{lei} or die 'BUG: {lei} missing';
+       my $curl_opt = delete $self->{"-curl-$url"} or
+                               die("BUG: $url curl options not prepared");
+       my $uri = pop @$curl_opt;
+       my $curl = PublicInbox::LeiCurl->new($lei, $self->{curl}) or return;
+       push @$curl, '-s', @$curl_opt;
+       my $cmd = $curl->for_uri($lei, $uri);
+       $lei->qerr("# $cmd");
+       my $rdr = { 2 => $lei->{2}, pgid => 0 };
+       my ($fh, $pid) = popen_rd($cmd, undef, $rdr);
+       grep(/\A--compressed\z/, @$curl) or
+               $fh = IO::Uncompress::Gunzip->new($fh, MultiStream => 1);
+       eval {
+               PublicInbox::MboxReader->mboxrd($fh,
+                                               $self->can('input_mbox_cb'),
+                                               $self, @args);
+       };
+       my $err = $@;
+       waitpid($pid, 0);
+       $? || $err and
+               $lei->child_error($? || 1, "@$cmd failed".$err ? " $err" : '');
+}
+
 sub input_path_url {
        my ($self, $input, @args) = @_;
        my $lei = $self->{lei};
@@ -92,6 +119,9 @@ sub input_path_url {
                                                $self->can('input_net_cb'),
                                        $self, @args);
                return;
+       } elsif ($input =~ m!\Ahttps?://!i) {
+               handle_http_input($self, $input, @args);
+               return;
        }
        if ($input =~ s!\A([a-z0-9]+):!!i) {
                $ifmt = lc($1);
@@ -129,6 +159,50 @@ EOM
        }
 }
 
+sub bad_http ($$;$) {
+       my ($lei, $url, $alt) = @_;
+       my $x = $alt ? "did you mean <$alt>?" : 'download and import manually';
+       $lei->fail("E: <$url> not recognized, $x");
+}
+
+sub prepare_http_input ($$$) {
+       my ($self, $lei, $url) = @_;
+       require URI;
+       require PublicInbox::MboxReader;
+       require PublicInbox::LeiCurl;
+       require IO::Uncompress::Gunzip;
+       $self->{curl} //= which('curl') or
+                               return $lei->fail("curl missing for <$url>");
+       my $uri = URI->new($url);
+       my $path = $uri->path;
+       my %qf = $uri->query_form;
+       my @curl_opt;
+       if ($path =~ m!/(?:t\.mbox\.gz|all\.mbox\.gz)\z!) {
+               # OK
+       } elsif ($path =~ m!/raw\z!) {
+               push @curl_opt, '--compressed';
+       # convert search query to mboxrd request since they require POST
+       # this is only intended for PublicInbox::WWW, and will false-positive
+       # on many other search engines... oh well
+       } elsif (defined $qf{'q'}) {
+               $qf{x} = 'm';
+               $uri->query_form(\%qf);
+               push @curl_opt, '-d', '';
+               $$uri ne $url and $lei->qerr(<<"");
+# <$url> rewritten to <$$uri> with HTTP POST
+
+       # try to provide hints for /$INBOX/$MSGID/T/ and /$INBOX/
+       } elsif ($path =~ s!/[tT]/\z!/t.mbox.gz! ||
+                       $path =~ s!/t\.atom\z!/t.mbox.gz! ||
+                       $path =~ s!/([^/]+\@[^/]+)/\z!/$1/raw!) {
+               $uri->path($path);
+               return bad_http($lei, $url, $$uri);
+       } else {
+               return bad_http($lei, $url);
+       }
+       $self->{"-curl-$url"} = [ @curl_opt, $uri ]; # for handle_http_input
+}
+
 sub prepare_inputs { # returns undef on error
        my ($self, $lei, $inputs) = @_;
        my $in_fmt = $lei->{opt}->{'in-format'};
@@ -156,6 +230,8 @@ sub prepare_inputs { # returns undef on error
                                        push @{$sync->{no}}, $input;
                                }
                        }
+               } elsif ($input_path =~ m!\Ahttps?://!i) {
+                       prepare_http_input($self, $lei, $input_path) or return;
                } elsif ($input_path =~ s/\A([a-z0-9]+)://is) {
                        my $ifmt = lc $1;
                        if (($in_fmt // $ifmt) ne $ifmt) {

diff --git a/t/lei-import-http.t b/t/lei-import-http.t
new file mode 100644 (file)
index 0000000..35cbf36
--- /dev/null
+++ b/t/lei-import-http.t
@@ -0,0 +1,43 @@
+#!perl -w
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict; use v5.10.1; use PublicInbox::TestCommon;
+use PublicInbox::Spawn qw(which);
+require_mods(qw(lei -httpd));
+which('curl') or plan skip_all => "curl required for $0";
+my ($ro_home, $cfg_path) = setup_public_inboxes;
+my ($tmpdir, $for_destroy) = tmpdir;
+my $sock = tcp_server;
+my $cmd = [ '-httpd', '-W0', "--stdout=$tmpdir/1", "--stderr=$tmpdir/2" ];
+my $env = { PI_CONFIG => $cfg_path };
+my $td = start_script($cmd, $env, { 3 => $sock }) or BAIL_OUT("-httpd $?");
+my $host_port = tcp_host_port($sock);
+undef $sock;
+test_lei({ tmpdir => $tmpdir }, sub {
+       my $url = "http://$host_port/t2";
+       for my $p (qw(bogus@x/t.mbox.gz bogus@x/raw ?q=noresultever)) {
+               ok(!lei('import', "$url/$p"), "/$p fails properly");
+       }
+       for my $p (qw(/ /T/ /t/ /t.atom)) {
+               ok(!lei('import', "$url/m\@example$p"), "/$p fails");
+               like($lei_err, qr/did you mean/, "gave hint for $p");
+       }
+       lei_ok 'import', "$url/testmessage\@example.com/raw";
+       lei_ok 'q', 'm:testmessage@example.com';
+       my $res = json_utf8->decode($lei_out);
+       is($res->[0]->{'m'}, 'testmessage@example.com', 'imported raw')
+               or diag explain($res);
+
+       lei_ok 'import', "$url/qp\@example.com/t.mbox.gz";
+       lei_ok 'q', 'm:qp@example.com';
+       $res = json_utf8->decode($lei_out);
+       is($res->[0]->{'m'}, 'qp@example.com', 'imported t.mbox.gz')
+               or diag explain($res);
+
+       lei_ok 'import', "$url/?q=s:boolean";
+       lei_ok 'q', 'm:20180720072141.GA15957@example';
+       $res = json_utf8->decode($lei_out);
+       is($res->[0]->{'m'}, '20180720072141.GA15957@example',
+                       'imported search result') or diag explain($res);
+});
+done_testing;