X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FLeiInput.pm;h=92d67715194b4f10bdc7750793b3e656a2fdf773;hb=26c426cf6fd9a3f44f11fa30e4824aeac574a6a0;hp=e416d3ed6e1777b428e3db9039c689fffc7ad7ac;hpb=ca7b99270e0574de573d29d6ba5b8a3bbdff82c5;p=public-inbox.git diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm index e416d3ed..92d67715 100644 --- a/lib/PublicInbox/LeiInput.pm +++ b/lib/PublicInbox/LeiInput.pm @@ -1,11 +1,44 @@ # Copyright (C) 2021 all contributors # License: AGPL-3.0+ -# parent class for LeiImport, LeiConvert +# parent class for LeiImport, LeiConvert, LeiIndex package PublicInbox::LeiInput; use strict; use v5.10.1; use PublicInbox::DS; +use PublicInbox::Spawn qw(which popen_rd); + +# JMAP RFC 8621 4.1.1 +# https://www.iana.org/assignments/imap-jmap-keywords/imap-jmap-keywords.xhtml +our @KW = (qw(seen answered flagged draft), # widely-compatible + qw(forwarded), # IMAP + Maildir + qw(phishing junk notjunk)); # rarely supported + +# note: RFC 8621 states "Users may add arbitrary keywords to an Email", +# but is it good idea? Stick to the system and reserved ones, for now. +# The widely-compatible ones map to IMAP system flags, Maildir flags +# and mbox Status/X-Status headers. +my %KW = map { $_ => 1 } @KW; +my $L_MAX = 244; # Xapian term limit - length('L') + +# RFC 8621, sec 2 (Mailboxes) a "label" for us is a JMAP Mailbox "name" +# "Servers MAY reject names that violate server policy" +my %ERR = ( + L => sub { + my ($label) = @_; + length($label) >= $L_MAX and + return "`$label' too long (must be <= $L_MAX)"; + $label =~ m{\A[a-z0-9_](?:[a-z0-9_\-\./\@,]*[a-z0-9])?\z}i ? + undef : "`$label' is invalid"; + }, + kw => sub { + my ($kw) = @_; + $KW{$kw} ? undef : <reads($fmt) or return $lei->fail("--$opt_key=$fmt unrecognized"); 1; @@ -37,6 +69,12 @@ error reading $name: $! # but no Content-Length or "From " escaping. # "git format-patch" also generates such files by default. $buf =~ s/\A[\r\n]*From [^\r\n]*\r?\n//s; + + # a user may feed just a body: git diff | lei rediff -U9 + if ($self->{-force_eml}) { + my $eml = PublicInbox::Eml->new($buf); + substr($buf, 0, 0) = "\n\n" if !$eml->{bdy}; + } $self->input_eml_cb(PublicInbox::Eml->new(\$buf), @args); } else { # prepare_inputs already validated $ifmt @@ -46,6 +84,28 @@ error reading $name: $! } } +# handles mboxrd endpoints described in Documentation/design_notes.txt +sub handle_http_input ($$@) { + my ($self, $url, @args) = @_; + my $lei = $self->{lei} or die 'BUG: {lei} missing'; + my $curl_opt = delete $self->{"-curl-$url"} or + die("BUG: $url curl options not prepared"); + my $uri = pop @$curl_opt; + my $curl = PublicInbox::LeiCurl->new($lei, $self->{curl}) or return; + push @$curl, '-s', @$curl_opt; + my $cmd = $curl->for_uri($lei, $uri); + $lei->qerr("# $cmd"); + my $rdr = { 2 => $lei->{2}, pgid => 0 }; + my ($fh, $pid) = popen_rd($cmd, undef, $rdr); + grep(/\A--compressed\z/, @$curl) or + $fh = IO::Uncompress::Gunzip->new($fh, MultiStream => 1); + eval { $self->input_fh('mboxrd', $fh, $url, @args) }; + my $err = $@; + waitpid($pid, 0); + $? || $err and + $lei->child_error($? || 1, "@$cmd failed".$err ? " $err" : ''); +} + sub input_path_url { my ($self, $input, @args) = @_; my $lei = $self->{lei}; @@ -53,11 +113,14 @@ sub input_path_url { # TODO auto-detect? if ($input =~ m!\Aimaps?://!i) { $lei->{net}->imap_each($input, $self->can('input_net_cb'), - $self, @args); + $self, @args); return; } elsif ($input =~ m!\A(?:nntps?|s?news)://!i) { $lei->{net}->nntp_each($input, $self->can('input_net_cb'), - $self, @args); + $self, @args); + return; + } elsif ($input =~ m!\Ahttps?://!i) { + handle_http_input($self, $input, @args); return; } if ($input =~ s!\A([a-z0-9]+):!!i) { @@ -86,34 +149,92 @@ sub input_path_url { $self->input_fh($ifmt, $mbl->{fh}, $input, @args); } elsif (-d _ && (-d "$input/cur" || -d "$input/new")) { return $lei->fail(<new->maildir_each_eml($input, - $self->can('input_maildir_cb'), - $self, @args); + my $mdr = PublicInbox::MdirReader->new; + if (my $pmd = $self->{pmd}) { + $mdr->maildir_each_file($input, + $pmd->can('each_mdir_fn'), + $pmd, @args); + } else { + $mdr->maildir_each_eml($input, + $self->can('input_maildir_cb'), + $self, @args); + } } else { $lei->fail("$input unsupported (TODO)"); } } +sub bad_http ($$;$) { + my ($lei, $url, $alt) = @_; + my $x = $alt ? "did you mean <$alt>?" : 'download and import manually'; + $lei->fail("E: <$url> not recognized, $x"); +} + +sub prepare_http_input ($$$) { + my ($self, $lei, $url) = @_; + require URI; + require PublicInbox::MboxReader; + require PublicInbox::LeiCurl; + require IO::Uncompress::Gunzip; + $self->{curl} //= which('curl') or + return $lei->fail("curl missing for <$url>"); + my $uri = URI->new($url); + my $path = $uri->path; + my %qf = $uri->query_form; + my @curl_opt; + if ($path =~ m!/(?:t\.mbox\.gz|all\.mbox\.gz)\z!) { + # OK + } elsif ($path =~ m!/raw\z!) { + push @curl_opt, '--compressed'; + # convert search query to mboxrd request since they require POST + # this is only intended for PublicInbox::WWW, and will false-positive + # on many other search engines... oh well + } elsif (defined $qf{'q'}) { + $qf{x} = 'm'; + $uri->query_form(\%qf); + push @curl_opt, '-d', ''; + $$uri ne $url and $lei->qerr(<<""); +# <$url> rewritten to <$$uri> with HTTP POST + + # try to provide hints for /$INBOX/$MSGID/T/ and /$INBOX/ + } elsif ($path =~ s!/[tT]/\z!/t.mbox.gz! || + $path =~ s!/t\.atom\z!/t.mbox.gz! || + $path =~ s!/([^/]+\@[^/]+)/\z!/$1/raw!) { + $uri->path($path); + return bad_http($lei, $url, $$uri); + } else { + return bad_http($lei, $url); + } + $self->{"-curl-$url"} = [ @curl_opt, $uri ]; # for handle_http_input +} + sub prepare_inputs { # returns undef on error my ($self, $lei, $inputs) = @_; my $in_fmt = $lei->{opt}->{'in-format'}; + my $sync = $lei->{opt}->{'mail-sync'} ? {} : undef; # using LeiMailSync if ($lei->{opt}->{stdin}) { @$inputs and return $lei->fail("--stdin and @$inputs do not mix"); check_input_format($lei) or return; push @$inputs, '/dev/stdin'; + push @{$sync->{no}}, '/dev/stdin' if $sync; } my $net = $lei->{net}; # NetWriter may be created by l2m - my (@f, @d); + my (@f, @md); # e.g. Maildir:/home/user/Mail/ or imaps://example.com/INBOX for my $input (@$inputs) { my $input_path = $input; if ($input =~ m!\A(?:imaps?|nntps?|s?news)://!i) { require PublicInbox::NetReader; $net //= PublicInbox::NetReader->new; - $net->add_url($input); + $net->add_url($input, $self->{-ls_ok}); + push @{$sync->{ok}}, $input if $sync; + } elsif ($input_path =~ m!\Ahttps?://!i) { # mboxrd.gz + # TODO: how would we detect r/w JMAP? + push @{$sync->{no}}, $input if $sync; + prepare_http_input($self, $lei, $input_path) or return; } elsif ($input_path =~ s/\A([a-z0-9]+)://is) { my $ifmt = lc $1; if (($in_fmt // $ifmt) ne $ifmt) { @@ -121,6 +242,11 @@ sub prepare_inputs { # returns undef on error --in-format=$in_fmt and `$ifmt:' conflict } + if ($ifmt =~ /\A(?:maildir|mh)\z/i) { + push @{$sync->{ok}}, $input if $sync; + } else { + push @{$sync->{no}}, $input if $sync; + } my $devfd = $lei->path_to_fd($input_path) // return; if ($devfd >= 0 || (-f $input_path || -p _)) { require PublicInbox::MboxLock; @@ -128,34 +254,48 @@ sub prepare_inputs { # returns undef on error PublicInbox::MboxReader->reads($ifmt) or return $lei->fail("$ifmt not supported"); } elsif (-d $input_path) { - require PublicInbox::MdirReader; $ifmt eq 'maildir' or return $lei->fail("$ifmt not supported"); + $sync and $input = 'maildir:'. + $lei->abs_path($input_path); + push @md, $input; } else { return $lei->fail("Unable to handle $input"); } - } elsif ($input =~ /\.(eml|patch)\z/i && -f $input) { + } elsif ($input =~ /\.(?:eml|patch)\z/i && -f $input) { lc($in_fmt//'eml') eq 'eml' or return $lei->fail(<<""); $input is `eml', not --in-format=$in_fmt require PublicInbox::Eml; + push @{$sync->{no}}, $input if $sync; } else { my $devfd = $lei->path_to_fd($input) // return; if ($devfd >= 0 || -f $input || -p _) { - push @f, $input - } elsif (-d $input) { - push @d, $input + push @{$sync->{no}}, $input if $sync; + push @f, $input; + } elsif (-d "$input/new" && -d "$input/cur") { + if ($sync) { + $input = $lei->abs_path($input); + push @{$sync->{ok}}, $input; + } + push @md, $input; } else { return $lei->fail("Unable to handle $input") } } } if (@f) { check_input_format($lei, \@f) or return } - if (@d) { # TODO: check for MH vs Maildir, here - require PublicInbox::MdirReader; + if ($sync && $sync->{no}) { + return $lei->fail(<<"") if !$sync->{ok}; +--mail-sync specified but no inputs support it + + # non-fatal if some inputs support support sync + $lei->err("# --mail-sync will only be used for @{$sync->{ok}}"); + $lei->err("# --mail-sync is not supported for: @{$sync->{no}}"); } if ($net) { - if (my $err = $net->errors) { + $net->{-can_die} = 1; + if (my $err = $net->errors($lei)) { return $lei->fail($err); } $net->{quiet} = $lei->{opt}->{quiet}; @@ -163,15 +303,29 @@ $input is `eml', not --in-format=$in_fmt $lei->{auth} //= PublicInbox::LeiAuth->new; $lei->{net} //= $net; } + if (scalar(@md)) { + require PublicInbox::MdirReader; + if ($self->can('pmdir_cb')) { + require PublicInbox::LeiPmdir; + $self->{pmd} = PublicInbox::LeiPmdir->new($lei, $self); + } + } $self->{inputs} = $inputs; } sub process_inputs { my ($self) = @_; + my $err; for my $input (@{$self->{inputs}}) { - $self->input_path_url($input); + eval { $self->input_path_url($input) }; + next unless $@; + $err = "$input: $@"; + last; } + # always commit first, even on error partial work is acceptable for + # lei my $wait = $self->{lei}->{sto}->ipc_do('done') if $self->{lei}->{sto}; + $self->{lei}->fail($err) if $err; } sub input_only_atfork_child { @@ -183,4 +337,33 @@ sub input_only_atfork_child { undef; } +# alias this as "net_merge_all_done" to use as an LeiAuth callback +sub input_only_net_merge_all_done { + my ($self) = @_; + $self->wq_io_do('process_inputs'); + $self->wq_close(1); +} + +# like Getopt::Long, but for +kw:FOO and -kw:FOO to prepare +# for update_xvmd -> update_vmd +sub vmd_mod_extract { + my $argv = $_[-1]; + my $vmd_mod = {}; + my @new_argv; + for my $x (@$argv) { + if ($x =~ /\A(\+|\-)(kw|L):(.+)\z/) { + my ($op, $pfx, $val) = ($1, $2, $3); + if (my $err = $ERR{$pfx}->($val)) { + push @{$vmd_mod->{err}}, $err; + } else { # set "+kw", "+L", "-L", "-kw" + push @{$vmd_mod->{$op.$pfx}}, $val; + } + } else { + push @new_argv, $x; + } + } + @$argv = @new_argv; + $vmd_mod; +} + 1;