X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FLeiInput.pm;h=dd40d83840c57cb2e790fd86d30699b8a11f60c5;hb=4b5a1b5787edee2a3b6cc10a3ccc5721f1414268;hp=4ff7a379b3c3800c96d4648ea629146d5b1627fc;hpb=55bed484ed5436b075c069ae9e42fb45890ad7e9;p=public-inbox.git

diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm
index 4ff7a379..dd40d838 100644
--- a/lib/PublicInbox/LeiInput.pm
+++ b/lib/PublicInbox/LeiInput.pm
@@ -7,6 +7,8 @@ use strict;
 use v5.10.1;
 use PublicInbox::DS;
 use PublicInbox::Spawn qw(which popen_rd);
+use PublicInbox::InboxWritable qw(eml_from_path);
+use PublicInbox::AutoReap;
 
 # JMAP RFC 8621 4.1.1
 # https://www.iana.org/assignments/imap-jmap-keywords/imap-jmap-keywords.xhtml
@@ -56,13 +58,19 @@ sub check_input_format ($;$) {
 	1;
 }
 
+sub input_mbox_cb { # base MboxReader callback
+	my ($eml, $self) = @_;
+	$eml->header_set($_) for (qw(Status X-Status));
+	$self->input_eml_cb($eml);
+}
+
 # import a single file handle of $name
 # Subclass must define ->input_eml_cb and ->input_mbox_cb
 sub input_fh {
 	my ($self, $ifmt, $fh, $name, @args) = @_;
 	if ($ifmt eq 'eml') {
 		my $buf = do { local $/; <$fh> } //
-			return $self->{lei}->child_error(1 << 8, <<"");
+			return $self->{lei}->child_error(0, <<"");
 error reading $name: $!
 
 		# mutt pipes single RFC822 messages with a "From " line,
@@ -95,15 +103,15 @@ sub handle_http_input ($$@) {
 	push @$curl, '-s', @$curl_opt;
 	my $cmd = $curl->for_uri($lei, $uri);
 	$lei->qerr("# $cmd");
-	my $rdr = { 2 => $lei->{2}, pgid => 0 };
-	my ($fh, $pid) = popen_rd($cmd, undef, $rdr);
+	my ($fh, $pid) = popen_rd($cmd, undef, { 2 => $lei->{2} });
+	my $ar = PublicInbox::AutoReap->new($pid);
 	grep(/\A--compressed\z/, @$curl) or
 		$fh = IO::Uncompress::Gunzip->new($fh, MultiStream => 1);
 	eval { $self->input_fh('mboxrd', $fh, $url, @args) };
 	my $err = $@;
-	waitpid($pid, 0);
+	$ar->join;
 	$? || $err and
-		$lei->child_error($? || 1, "@$cmd failed".$err ? " $err" : '');
+		$lei->child_error($?, "@$cmd failed".$err ? " $err" : '');
 }
 
 sub input_path_url {
@@ -123,10 +131,24 @@ sub input_path_url {
 		handle_http_input($self, $input, @args);
 		return;
 	}
+
+	# local-only below
+	my $ifmt_pfx = '';
 	if ($input =~ s!\A([a-z0-9]+):!!i) {
+		$ifmt_pfx = "$1:";
 		$ifmt = lc($1);
 	} elsif ($input =~ /\.(?:patch|eml)\z/i) {
 		$ifmt = 'eml';
+	} elsif (-f $input && $input =~ m{\A(?:.+)/(?:new|cur)/([^/]+)\z}) {
+		my $bn = $1;
+		my $fl = PublicInbox::MdirReader::maildir_basename_flags($bn);
+		return if index($fl, 'T') >= 0;
+		return $self->pmdir_cb($input, $fl) if $self->can('pmdir_cb');
+		my $eml = eml_from_path($input) or return
+			$lei->qerr("# $input not readable");
+		my $kw = PublicInbox::MdirReader::flags2kw($fl);
+		$self->can('input_maildir_cb')->($input, $kw, $eml, $self);
+		return;
 	}
 	my $devfd = $lei->path_to_fd($input) // return;
 	if ($devfd >= 0) {
@@ -151,14 +173,26 @@ sub input_path_url {
 		return $lei->fail(<<EOM) if $ifmt && $ifmt ne 'maildir';
 $input appears to be a maildir, not $ifmt
 EOM
-		PublicInbox::MdirReader->new->maildir_each_eml($input,
-					$self->can('input_maildir_cb'),
-					$self, @args);
+		my $mdr = PublicInbox::MdirReader->new;
+		if (my $pmd = $self->{pmd}) {
+			$mdr->maildir_each_file($input,
+						$pmd->can('each_mdir_fn'),
+						$pmd, @args);
+		} else {
+			$mdr->maildir_each_eml($input,
+						$self->can('input_maildir_cb'),
+						$self, @args);
+		}
+	} elsif ($self->{missing_ok} && !-e $input) { # don't ->fail
+		$self->folder_missing("$ifmt:$input");
 	} else {
-		$lei->fail("$input unsupported (TODO)");
+		$lei->fail("$ifmt_pfx$input unsupported (TODO)");
 	}
 }
 
+# subclasses should overrride this (see LeiRefreshMailSync)
+sub folder_missing { die "BUG: ->folder_missing undefined for $_[0]" }
+
 sub bad_http ($$;$) {
 	my ($lei, $url, $alt) = @_;
 	my $x = $alt ? "did you mean <$alt>?" : 'download and import manually';
@@ -207,6 +241,7 @@ sub prepare_inputs { # returns undef on error
 	my ($self, $lei, $inputs) = @_;
 	my $in_fmt = $lei->{opt}->{'in-format'};
 	my $sync = $lei->{opt}->{'mail-sync'} ? {} : undef; # using LeiMailSync
+	my $may_sync = $sync || $self->{-mail_sync};
 	if ($lei->{opt}->{stdin}) {
 		@$inputs and return
 			$lei->fail("--stdin and @$inputs do not mix");
@@ -215,14 +250,14 @@ sub prepare_inputs { # returns undef on error
 		push @{$sync->{no}}, '/dev/stdin' if $sync;
 	}
 	my $net = $lei->{net}; # NetWriter may be created by l2m
-	my (@f, @d);
+	my (@f, @md);
 	# e.g. Maildir:/home/user/Mail/ or imaps://example.com/INBOX
 	for my $input (@$inputs) {
 		my $input_path = $input;
 		if ($input =~ m!\A(?:imaps?|nntps?|s?news)://!i) {
 			require PublicInbox::NetReader;
 			$net //= PublicInbox::NetReader->new;
-			$net->add_url($input);
+			$net->add_url($input, $self->{-ls_ok});
 			push @{$sync->{ok}}, $input if $sync;
 		} elsif ($input_path =~ m!\Ahttps?://!i) { # mboxrd.gz
 			# TODO: how would we detect r/w JMAP?
@@ -247,47 +282,70 @@ sub prepare_inputs { # returns undef on error
 				PublicInbox::MboxReader->reads($ifmt) or return
 					$lei->fail("$ifmt not supported");
 			} elsif (-d $input_path) {
-				require PublicInbox::MdirReader;
 				$ifmt eq 'maildir' or return
 					$lei->fail("$ifmt not supported");
-				$sync and $input = 'maildir:'.
+				$may_sync and $input = 'maildir:'.
+						$lei->abs_path($input_path);
+				push @md, $input;
+			} elsif ($self->{missing_ok} && !-e _) {
+				# for "lei rm-watch" on missing Maildir
+				$may_sync and $input = 'maildir:'.
 						$lei->abs_path($input_path);
 			} else {
-				return $lei->fail("Unable to handle $input");
+				my $m = "Unable to handle $input";
+				$input =~ /\A(?:L|kw):/ and
+					$m .= ", did you mean +$input?";
+				return $lei->fail($m);
 			}
 		} elsif ($input =~ /\.(?:eml|patch)\z/i && -f $input) {
 			lc($in_fmt//'eml') eq 'eml' or return $lei->fail(<<"");
 $input is `eml', not --in-format=$in_fmt
 
-			require PublicInbox::Eml;
 			push @{$sync->{no}}, $input if $sync;
+		} elsif (-f $input && $input =~ m{\A(.+)/(new|cur)/([^/]+)\z}) {
+			# single file in a Maildir
+			my ($mdir, $nc, $bn) = ($1, $2, $3);
+			my $other = $mdir . ($nc eq 'new' ? '/cur' : '/new');
+			return $lei->fail(<<EOM) if !-d $other;
+No `$other' directory for `$input'
+EOM
+			lc($in_fmt//'eml') eq 'eml' or return $lei->fail(<<"");
+$input is `eml', not --in-format=$in_fmt
+
+			if ($sync) {
+				$input = $lei->abs_path($mdir) . "/$nc/$bn";
+				push @{$sync->{ok}}, $input if $sync;
+			}
+			require PublicInbox::MdirReader;
 		} else {
 			my $devfd = $lei->path_to_fd($input) // return;
 			if ($devfd >= 0 || -f $input || -p _) {
 				push @{$sync->{no}}, $input if $sync;
 				push @f, $input;
-			} elsif (-d $input) {
-				if ($sync) {
-					$input = $lei->abs_path($input);
-					push @{$sync->{ok}}, $input;
+			} elsif (-d "$input/new" && -d "$input/cur") {
+				if ($may_sync) {
+					$input = 'maildir:'.
+						$lei->abs_path($input);
+					push @{$sync->{ok}}, $input if $sync;
 				}
-				push @d, $input;
+				push @md, $input;
+			} elsif ($self->{missing_ok} && !-e $input) {
+				# for lei rm-watch
+				$may_sync and $input = 'maildir:'.
+						$lei->abs_path($input);
 			} else {
 				return $lei->fail("Unable to handle $input")
 			}
 		}
 	}
 	if (@f) { check_input_format($lei, \@f) or return }
-	if (@d) { # TODO: check for MH vs Maildir, here
-		require PublicInbox::MdirReader;
-	}
 	if ($sync && $sync->{no}) {
 		return $lei->fail(<<"") if !$sync->{ok};
 --mail-sync specified but no inputs support it
 
 		# non-fatal if some inputs support support sync
-		$lei->err("# --mail-sync will only be used for @{$sync->{ok}}");
-		$lei->err("# --mail-sync is not supported for: @{$sync->{no}}");
+		warn("# --mail-sync will only be used for @{$sync->{ok}}\n");
+		warn("# --mail-sync is not supported for: @{$sync->{no}}\n");
 	}
 	if ($net) {
 		$net->{-can_die} = 1;
@@ -299,6 +357,20 @@ $input is `eml', not --in-format=$in_fmt
 		$lei->{auth} //= PublicInbox::LeiAuth->new;
 		$lei->{net} //= $net;
 	}
+	if (scalar(@md)) {
+		require PublicInbox::MdirReader;
+		if ($self->can('pmdir_cb')) {
+			require PublicInbox::LeiPmdir;
+			$self->{pmd} = PublicInbox::LeiPmdir->new($lei, $self);
+		}
+
+		# start watching Maildirs ASAP
+		if ($may_sync && $lei->{sto}) {
+			grep(!m!\Amaildir:/!i, @md) and die "BUG: @md (no pfx)";
+			$lei->lms(1)->lms_write_prepare->add_folders(@md);
+			$lei->refresh_watches;
+		}
+	}
 	$self->{inputs} = $inputs;
 }
 
@@ -313,7 +385,7 @@ sub process_inputs {
 	}
 	# always commit first, even on error partial work is acceptable for
 	# lei <import|tag|convert>
-	my $wait = $self->{lei}->{sto}->ipc_do('done') if $self->{lei}->{sto};
+	my $wait = $self->{lei}->{sto}->wq_do('done') if $self->{lei}->{sto};
 	$self->{lei}->fail($err) if $err;
 }
 
@@ -335,6 +407,7 @@ sub input_only_net_merge_all_done {
 
 # like Getopt::Long, but for +kw:FOO and -kw:FOO to prepare
 # for update_xvmd -> update_vmd
+# returns something like { "+L" => [ @Labels ], ... }
 sub vmd_mod_extract {
 	my $argv = $_[-1];
 	my $vmd_mod = {};