+ $cb->(undef, $fh, $self->can('input_mbox_cb'), $self, @args);
+ }
+}
+
+# handles mboxrd endpoints described in Documentation/design_notes.txt
+sub handle_http_input ($$@) {
+ my ($self, $url, @args) = @_;
+ my $lei = $self->{lei} or die 'BUG: {lei} missing';
+ my $curl_opt = delete $self->{"-curl-$url"} or
+ die("BUG: $url curl options not prepared");
+ my $uri = pop @$curl_opt;
+ my $curl = PublicInbox::LeiCurl->new($lei, $self->{curl}) or return;
+ push @$curl, '-s', @$curl_opt;
+ my $cmd = $curl->for_uri($lei, $uri);
+ $lei->qerr("# $cmd");
+ my $rdr = { 2 => $lei->{2}, pgid => 0 };
+ my ($fh, $pid) = popen_rd($cmd, undef, $rdr);
+ grep(/\A--compressed\z/, @$curl) or
+ $fh = IO::Uncompress::Gunzip->new($fh, MultiStream => 1);
+ eval { $self->input_fh('mboxrd', $fh, $url, @args) };
+ my $err = $@;
+ waitpid($pid, 0);
+ $? || $err and
+ $lei->child_error($? || 1, "@$cmd failed".$err ? " $err" : '');
+}
+
+sub input_path_url {
+ my ($self, $input, @args) = @_;
+ my $lei = $self->{lei};
+ my $ifmt = lc($lei->{opt}->{'in-format'} // '');
+ # TODO auto-detect?
+ if ($input =~ m!\Aimaps?://!i) {
+ $lei->{net}->imap_each($input, $self->can('input_net_cb'),
+ $self, @args);
+ return;
+ } elsif ($input =~ m!\A(?:nntps?|s?news)://!i) {
+ $lei->{net}->nntp_each($input, $self->can('input_net_cb'),
+ $self, @args);
+ return;
+ } elsif ($input =~ m!\Ahttps?://!i) {
+ handle_http_input($self, $input, @args);
+ return;
+ }
+ if ($input =~ s!\A([a-z0-9]+):!!i) {
+ $ifmt = lc($1);
+ } elsif ($input =~ /\.(?:patch|eml)\z/i) {
+ $ifmt = 'eml';
+ }
+ my $devfd = $lei->path_to_fd($input) // return;
+ if ($devfd >= 0) {
+ $self->input_fh($ifmt, $lei->{$devfd}, $input, @args);
+ } elsif (-f $input && $ifmt eq 'eml') {
+ open my $fh, '<', $input or
+ return $lei->fail("open($input): $!");
+ $self->input_fh($ifmt, $fh, $input, @args);
+ } elsif (-f _) {
+ my $m = $lei->{opt}->{'lock'} //
+ PublicInbox::MboxLock->defaults;
+ my $mbl = PublicInbox::MboxLock->acq($input, 0, $m);
+ my $zsfx = PublicInbox::MboxReader::zsfx($input);
+ if ($zsfx) {
+ my $in = delete $mbl->{fh};
+ $mbl->{fh} =
+ PublicInbox::MboxReader::zsfxcat($in, $zsfx, $lei);
+ }
+ local $PublicInbox::DS::in_loop = 0 if $zsfx; # dwaitpid
+ $self->input_fh($ifmt, $mbl->{fh}, $input, @args);
+ } elsif (-d _ && (-d "$input/cur" || -d "$input/new")) {
+ return $lei->fail(<<EOM) if $ifmt && $ifmt ne 'maildir';
+$input appears to be a maildir, not $ifmt
+EOM
+ my $mdr = PublicInbox::MdirReader->new;
+ if (my $pmd = $self->{pmd}) {
+ $mdr->maildir_each_file($input,
+ $pmd->can('each_mdir_fn'),
+ $pmd, @args);
+ } else {
+ $mdr->maildir_each_eml($input,
+ $self->can('input_maildir_cb'),
+ $self, @args);
+ }
+ } else {
+ $lei->fail("$input unsupported (TODO)");
+ }
+}
+
+sub bad_http ($$;$) {
+ my ($lei, $url, $alt) = @_;
+ my $x = $alt ? "did you mean <$alt>?" : 'download and import manually';
+ $lei->fail("E: <$url> not recognized, $x");
+}
+
+sub prepare_http_input ($$$) {
+ my ($self, $lei, $url) = @_;
+ require URI;
+ require PublicInbox::MboxReader;
+ require PublicInbox::LeiCurl;
+ require IO::Uncompress::Gunzip;
+ $self->{curl} //= which('curl') or
+ return $lei->fail("curl missing for <$url>");
+ my $uri = URI->new($url);
+ my $path = $uri->path;
+ my %qf = $uri->query_form;
+ my @curl_opt;
+ if ($path =~ m!/(?:t\.mbox\.gz|all\.mbox\.gz)\z!) {
+ # OK
+ } elsif ($path =~ m!/raw\z!) {
+ push @curl_opt, '--compressed';
+ # convert search query to mboxrd request since they require POST
+ # this is only intended for PublicInbox::WWW, and will false-positive
+ # on many other search engines... oh well
+ } elsif (defined $qf{'q'}) {
+ $qf{x} = 'm';
+ $uri->query_form(\%qf);
+ push @curl_opt, '-d', '';
+ $$uri ne $url and $lei->qerr(<<"");
+# <$url> rewritten to <$$uri> with HTTP POST
+
+ # try to provide hints for /$INBOX/$MSGID/T/ and /$INBOX/
+ } elsif ($path =~ s!/[tT]/\z!/t.mbox.gz! ||
+ $path =~ s!/t\.atom\z!/t.mbox.gz! ||
+ $path =~ s!/([^/]+\@[^/]+)/\z!/$1/raw!) {
+ $uri->path($path);
+ return bad_http($lei, $url, $$uri);
+ } else {
+ return bad_http($lei, $url);