From a00fdf39b4ec4ed045f28c29c79c8dcb9abac836 Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Mon, 22 Mar 2021 07:53:57 +0000 Subject: [PATCH] lei: share input code between convert and import These commands accept mail the same way, and this forces us to maintain consistent input format support between commands. We'll be using this for "lei mark", too. --- MANIFEST | 1 + lib/PublicInbox/LEI.pm | 17 ------- lib/PublicInbox/LeiConvert.pm | 60 +++---------------------- lib/PublicInbox/LeiImport.pm | 57 ++--------------------- lib/PublicInbox/LeiInput.pm | 85 +++++++++++++++++++++++++++++++++++ 5 files changed, 94 insertions(+), 126 deletions(-) create mode 100644 lib/PublicInbox/LeiInput.pm diff --git a/MANIFEST b/MANIFEST index b6b4a3ab..df8440ef 100644 --- a/MANIFEST +++ b/MANIFEST @@ -187,6 +187,7 @@ lib/PublicInbox/LeiDedupe.pm lib/PublicInbox/LeiExternal.pm lib/PublicInbox/LeiHelp.pm lib/PublicInbox/LeiImport.pm +lib/PublicInbox/LeiInput.pm lib/PublicInbox/LeiMirror.pm lib/PublicInbox/LeiOverview.pm lib/PublicInbox/LeiP2q.pm diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm index 9e3bb9b7..0bd52a46 100644 --- a/lib/PublicInbox/LEI.pm +++ b/lib/PublicInbox/LEI.pm @@ -419,23 +419,6 @@ sub fail ($$;$) { undef; } -sub check_input_format ($;$) { - my ($self, $files) = @_; - my $opt_key = 'in-format'; - my $fmt = $self->{opt}->{$opt_key}; - if (!$fmt) { - my $err = $files ? "regular file(s):\n@$files" : '--stdin'; - return fail($self, "--$opt_key unset for $err"); - } - require PublicInbox::MboxLock if $files; - require PublicInbox::MboxReader; - return 1 if $fmt eq 'eml'; - # XXX: should this handle {gz,bz2,xz}? that's currently in LeiToMail - PublicInbox::MboxReader->can($fmt) or - return fail($self, "--$opt_key=$fmt unrecognized"); - 1; -} - sub out ($;@) { my $self = shift; return if print { $self->{1} // return } @_; # likely diff --git a/lib/PublicInbox/LeiConvert.pm b/lib/PublicInbox/LeiConvert.pm index 8d3b221a..0aa13229 100644 --- a/lib/PublicInbox/LeiConvert.pm +++ b/lib/PublicInbox/LeiConvert.pm @@ -5,7 +5,7 @@ package PublicInbox::LeiConvert; use strict; use v5.10.1; -use parent qw(PublicInbox::IPC); +use parent qw(PublicInbox::IPC PublicInbox::LeiInput); use PublicInbox::Eml; use PublicInbox::LeiStore; use PublicInbox::LeiOverview; @@ -79,64 +79,14 @@ sub do_convert { # via wq_do sub lei_convert { # the main "lei convert" method my ($lei, @inputs) = @_; - my $opt = $lei->{opt}; - $opt->{kw} //= 1; + $lei->{opt}->{kw} //= 1; + $lei->{opt}->{dedupe} //= 'none'; my $self = $lei->{cnv} = bless {}, __PACKAGE__; - my $in_fmt = $opt->{'in-format'}; - my (@f, @d); - $opt->{dedupe} //= 'none'; my $ovv = PublicInbox::LeiOverview->new($lei, 'out-format'); $lei->{l2m} or return $lei->fail("output not specified or is not a mail destination"); - my $net = $lei->{net}; # NetWriter may be created by l2m - $opt->{augment} = 1 unless $ovv->{dst} eq '/dev/stdout'; - if ($opt->{stdin}) { - @inputs and return $lei->fail("--stdin and @inputs do not mix"); - $lei->check_input_format(undef) or return; - $self->{0} = $lei->{0}; - } - # e.g. Maildir:/home/user/Mail/ or imaps://example.com/INBOX - for my $input (@inputs) { - my $input_path = $input; - if ($input =~ m!\A(?:imaps?|nntps?|s?news)://!i) { - require PublicInbox::NetReader; - $net //= PublicInbox::NetReader->new; - $net->add_url($input); - } elsif ($input_path =~ s/\A([a-z0-9]+)://is) { - my $ifmt = lc $1; - if (($in_fmt // $ifmt) ne $ifmt) { - return $lei->fail(<<""); ---in-format=$in_fmt and `$ifmt:' conflict - - } - if (-f $input_path) { - require PublicInbox::MboxLock; - require PublicInbox::MboxReader; - PublicInbox::MboxReader->can($ifmt) or return - $lei->fail("$ifmt not supported"); - } elsif (-d _) { - require PublicInbox::MdirReader; - $ifmt eq 'maildir' or return - $lei->fail("$ifmt not supported"); - } else { - return $lei->fail("Unable to handle $input"); - } - } elsif (-f $input) { push @f, $input } - elsif (-d _) { push @d, $input } - else { return $lei->fail("Unable to handle $input") } - } - if (@f) { $lei->check_input_format(\@f) or return } - if (@d) { # TODO: check for MH vs Maildir, here - require PublicInbox::MdirReader; - } - $self->{inputs} = \@inputs; - if ($net) { - if (my $err = $net->errors) { - return $lei->fail($err); - } - $net->{quiet} = $opt->{quiet}; - $lei->{net} //= $net; - } + $lei->{opt}->{augment} = 1 unless $ovv->{dst} eq '/dev/stdout'; + $self->prepare_inputs($lei, \@inputs) or return; my $op = $lei->workers_start($self, 'lei_convert', 1, { '' => [ $lei->can('dclose'), $lei ] }); diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm index 0e2a96e8..e769fba8 100644 --- a/lib/PublicInbox/LeiImport.pm +++ b/lib/PublicInbox/LeiImport.pm @@ -5,7 +5,7 @@ package PublicInbox::LeiImport; use strict; use v5.10.1; -use parent qw(PublicInbox::IPC); +use parent qw(PublicInbox::IPC PublicInbox::LeiInput); use PublicInbox::Eml; use PublicInbox::PktOp qw(pkt_do); @@ -67,60 +67,9 @@ sub lei_import { # the main "lei import" method my ($lei, @inputs) = @_; my $sto = $lei->_lei_store(1); $sto->write_prepare($lei); - my ($net, @f, @d); $lei->{opt}->{kw} //= 1; - my $self = $lei->{imp} = bless { inputs => \@inputs }, __PACKAGE__; - if ($lei->{opt}->{stdin}) { - @inputs and return $lei->fail("--stdin and @inputs do not mix"); - $lei->check_input_format or return; - $self->{0} = $lei->{0}; - } - - my $fmt = $lei->{opt}->{'in-format'}; - # e.g. Maildir:/home/user/Mail/ or imaps://example.com/INBOX - for my $input (@inputs) { - my $input_path = $input; - if ($input =~ m!\A(?:imaps?|nntps?|s?news)://!i) { - require PublicInbox::NetReader; - $net //= PublicInbox::NetReader->new; - $net->add_url($input); - } elsif ($input_path =~ s/\A([a-z0-9]+)://is) { - my $ifmt = lc $1; - if (($fmt // $ifmt) ne $ifmt) { - return $lei->fail(<<""); ---in-format=$fmt and `$ifmt:' conflict - - } - if (-f $input_path) { - require PublicInbox::MboxLock; - require PublicInbox::MboxReader; - PublicInbox::MboxReader->can($ifmt) or return - $lei->fail("$ifmt not supported"); - } elsif (-d _) { - require PublicInbox::MdirReader; - $ifmt eq 'maildir' or return - $lei->fail("$ifmt not supported"); - } else { - return $lei->fail("Unable to handle $input"); - } - } elsif (-f $input) { push @f, $input - } elsif (-d _) { push @d, $input - } else { return $lei->fail("Unable to handle $input") } - } - if (@f) { $lei->check_input_format(\@f) or return } - if (@d) { # TODO: check for MH vs Maildir, here - require PublicInbox::MdirReader; - } - $self->{inputs} = \@inputs; - if ($net) { - if (my $err = $net->errors) { - return $lei->fail($err); - } - $net->{quiet} = $lei->{opt}->{quiet}; - $lei->{net} = $net; - require PublicInbox::LeiAuth; - $lei->{auth} = PublicInbox::LeiAuth->new; - } + my $self = $lei->{imp} = bless {}, __PACKAGE__; + $self->prepare_inputs($lei, \@inputs) or return; import_start($lei); } diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm new file mode 100644 index 00000000..89585a52 --- /dev/null +++ b/lib/PublicInbox/LeiInput.pm @@ -0,0 +1,85 @@ +# Copyright (C) 2021 all contributors +# License: AGPL-3.0+ + +# parent class for LeiImport, LeiConvert +package PublicInbox::LeiInput; +use strict; +use v5.10.1; + +sub check_input_format ($;$) { + my ($lei, $files) = @_; + my $opt_key = 'in-format'; + my $fmt = $lei->{opt}->{$opt_key}; + if (!$fmt) { + my $err = $files ? "regular file(s):\n@$files" : '--stdin'; + return $lei->fail("--$opt_key unset for $err"); + } + require PublicInbox::MboxLock if $files; + require PublicInbox::MboxReader; + return 1 if $fmt eq 'eml'; + # XXX: should this handle {gz,bz2,xz}? that's currently in LeiToMail + PublicInbox::MboxReader->can($fmt) or + return $lei->fail("--$opt_key=$fmt unrecognized"); + 1; +} + + +sub prepare_inputs { + my ($self, $lei, $inputs) = @_; + my $in_fmt = $lei->{opt}->{'in-format'}; + if ($lei->{opt}->{stdin}) { + @$inputs and return + $lei->fail("--stdin and @$inputs do not mix"); + check_input_format($lei) or return; + $self->{0} = $lei->{0}; + } + my $net = $lei->{net}; # NetWriter may be created by l2m + my $fmt = $lei->{opt}->{'in-format'}; + my (@f, @d); + # e.g. Maildir:/home/user/Mail/ or imaps://example.com/INBOX + for my $input (@$inputs) { + my $input_path = $input; + if ($input =~ m!\A(?:imaps?|nntps?|s?news)://!i) { + require PublicInbox::NetReader; + $net //= PublicInbox::NetReader->new; + $net->add_url($input); + } elsif ($input_path =~ s/\A([a-z0-9]+)://is) { + my $ifmt = lc $1; + if (($in_fmt // $ifmt) ne $ifmt) { + return $lei->fail(<<""); +--in-format=$in_fmt and `$ifmt:' conflict + + } + if (-f $input_path) { + require PublicInbox::MboxLock; + require PublicInbox::MboxReader; + PublicInbox::MboxReader->can($ifmt) or return + $lei->fail("$ifmt not supported"); + } elsif (-d _) { + require PublicInbox::MdirReader; + $ifmt eq 'maildir' or return + $lei->fail("$ifmt not supported"); + } else { + return $lei->fail("Unable to handle $input"); + } + } elsif (-f $input) { push @f, $input } + elsif (-d _) { push @d, $input } + else { return $lei->fail("Unable to handle $input") } + } + if (@f) { check_input_format($lei, \@f) or return } + if (@d) { # TODO: check for MH vs Maildir, here + require PublicInbox::MdirReader; + } + if ($net) { + if (my $err = $net->errors) { + return $lei->fail($err); + } + $net->{quiet} = $lei->{opt}->{quiet}; + require PublicInbox::LeiAuth; + $lei->{auth} //= PublicInbox::LeiAuth->new; + $lei->{net} //= $net; + } + $self->{inputs} = $inputs; +} + +1; -- 2.44.0