X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FInboxWritable.pm;h=982ad6e59d6afc43c265c0b4e454878985238094;hb=af0b0fb7a454470a32c452119d0392e0dedb3fe1;hp=f00141d286e42a62e1f7f3d00901ea72d5fcf4c3;hpb=c477bdd8a80eecc319b680764edfb24bd12cb7b2;p=public-inbox.git diff --git a/lib/PublicInbox/InboxWritable.pm b/lib/PublicInbox/InboxWritable.pm index f00141d2..982ad6e5 100644 --- a/lib/PublicInbox/InboxWritable.pm +++ b/lib/PublicInbox/InboxWritable.pm @@ -1,14 +1,15 @@ -# Copyright (C) 2018 all contributors +# Copyright (C) 2018-2021 all contributors # License: AGPL-3.0+ # Extends read-only Inbox for writing package PublicInbox::InboxWritable; use strict; -use warnings; -use base qw(PublicInbox::Inbox); +use v5.10.1; +use parent qw(PublicInbox::Inbox Exporter); use PublicInbox::Import; -use PublicInbox::Filter::Base; -*REJECT = *PublicInbox::Filter::Base::REJECT; +use PublicInbox::Filter::Base qw(REJECT); +use Errno qw(ENOENT); +our @EXPORT_OK = qw(eml_from_path warn_ignore_cb); use constant { PERM_UMASK => 0, @@ -20,23 +21,51 @@ use constant { sub new { my ($class, $ibx, $creat_opt) = @_; + return $ibx if ref($ibx) eq $class; my $self = bless $ibx, $class; # TODO: maybe stop supporting this if ($creat_opt) { # for { nproc => $N } $self->{-creat_opt} = $creat_opt; - init_inbox($self) if ($self->{version} || 1) == 1; + init_inbox($self) if $self->version == 1; } $self; } +sub assert_usable_dir { + my ($self) = @_; + my $dir = $self->{inboxdir}; + return $dir if defined($dir) && $dir ne ''; + die "no inboxdir defined for $self->{name}\n"; +} + +sub _init_v1 { + my ($self, $skip_artnum) = @_; + if (defined($self->{indexlevel}) || defined($skip_artnum)) { + require PublicInbox::SearchIdx; + require PublicInbox::Msgmap; + my $sidx = PublicInbox::SearchIdx->new($self, 1); # just create + $sidx->begin_txn_lazy; + my $mm = PublicInbox::Msgmap->new($self->{inboxdir}, 1); + if (defined $skip_artnum) { + $mm->{dbh}->begin_work; + $mm->skip_artnum($skip_artnum); + $mm->{dbh}->commit; + } + undef $mm; # ->created_at set + $sidx->commit_txn_lazy; + } else { + open my $fh, '>>', "$self->{inboxdir}/ssoma.lock" or + die "$self->{inboxdir}/ssoma.lock: $!\n"; + } +} + sub init_inbox { my ($self, $shards, $skip_epoch, $skip_artnum) = @_; - # TODO: honor skip_artnum - my $v = $self->{version} || 1; - if ($v == 1) { - my $dir = $self->{mainrepo} or die "no mainrepo in inbox\n"; + if ($self->version == 1) { + my $dir = assert_usable_dir($self); PublicInbox::Import::init_bare($dir); + $self->with_umask(\&_init_v1, $self, $skip_artnum); } else { my $v2w = importer($self); $v2w->init_inbox($shards, $skip_epoch, $skip_artnum); @@ -45,22 +74,20 @@ sub init_inbox { sub importer { my ($self, $parallel) = @_; - $self->{-importer} ||= do { - my $v = $self->{version} || 1; - if ($v == 2) { - eval { require PublicInbox::V2Writable }; - die "v2 not supported: $@\n" if $@; - my $opt = $self->{-creat_opt}; - my $v2w = PublicInbox::V2Writable->new($self, $opt); - $v2w->{parallel} = $parallel; - $v2w; - } elsif ($v == 1) { - my @arg = (undef, undef, undef, $self); - PublicInbox::Import->new(@arg); - } else { - $! = 78; # EX_CONFIG 5.3.5 local configuration error - die "unsupported inbox version: $v\n"; - } + my $v = $self->version; + if ($v == 2) { + eval { require PublicInbox::V2Writable }; + die "v2 not supported: $@\n" if $@; + my $opt = $self->{-creat_opt}; + my $v2w = PublicInbox::V2Writable->new($self, $opt); + $v2w->{parallel} = $parallel if defined $parallel; + $v2w; + } elsif ($v == 1) { + my @arg = (undef, undef, undef, $self); + PublicInbox::Import->new(@arg); + } else { + $! = 78; # EX_CONFIG 5.3.5 local configuration error + die "unsupported inbox version: $v\n"; } } @@ -71,11 +98,11 @@ sub filter { # v2 keeps msgmap open, which causes conflicts for filters # such as PublicInbox::Filter::RubyLang which overload msgmap # for a predictable serial number. - if ($im && ($self->{version} || 1) >= 2 && $self->{altid}) { + if ($im && $self->version >= 2 && $self->{altid}) { $im->done; } - my @args = (-inbox => $self); + my @args = (ibx => $self); # basic line splitting, only # Perhaps we can have proper quote splitting one day... ($f, @args) = split(/\s+/, $f) if $f =~ /\s+/; @@ -107,19 +134,14 @@ sub is_maildir_path ($) { (is_maildir_basename($p[-1]) && -f $path) ? 1 : 0; } -sub maildir_path_load ($) { +sub eml_from_path ($) { my ($path) = @_; if (open my $fh, '<', $path) { - local $/; - my $str = <$fh>; - $str or return; - return PublicInbox::MIME->new(\$str); - } elsif ($!{ENOENT}) { - # common with Maildir - return; - } else { - warn "failed to open $path: $!\n"; - return; + my $str = do { local $/; <$fh> } or return; + PublicInbox::Eml->new(\$str); + } else { # ENOENT is common with Maildir + warn "failed to open $path: $!\n" if $! != ENOENT; + undef; } } @@ -134,7 +156,7 @@ sub import_maildir { opendir my $dh, "$dir/$sub" or die "opendir $dir/$sub: $!\n"; while (defined(my $fn = readdir($dh))) { next unless is_maildir_basename($fn); - my $mime = maildir_file_load("$dir/$fn") or next; + my $mime = eml_from_path("$dir/$fn") or next; if (my $filter = $self->filter($im)) { my $ret = $filter->scrub($mime) or return; @@ -153,12 +175,12 @@ my $from_strict = qr/^From \S+ +\S+ \S+ +\S+ [^:]+:[^:]+:[^:]+ [^:]+/; sub mb_add ($$$$) { my ($im, $variant, $filter, $msg) = @_; $$msg =~ s/(\r?\n)+\z/$1/s; - my $mime = PublicInbox::MIME->new($msg); if ($variant eq 'mboxrd') { - $$msg =~ s/^>(>*From )/$1/sm; + $$msg =~ s/^>(>*From )/$1/gms; } elsif ($variant eq 'mboxo') { - $$msg =~ s/^>From /From /sm; + $$msg =~ s/^>From /From /gms; } + my $mime = PublicInbox::Eml->new($msg); if ($filter) { my $ret = $filter->scrub($mime) or return; return if $ret == REJECT(); @@ -236,9 +258,9 @@ sub _umask_for { } sub with_umask { - my ($self, $cb) = @_; - my $old = umask $self->{umask}; - my $rv = eval { $cb->() }; + my ($self, $cb, @arg) = @_; + my $old = umask($self->{umask} //= umask_prepare($self)); + my $rv = eval { $cb->(@arg) }; my $err = $@; umask $old; die $err if $err; @@ -248,8 +270,40 @@ sub with_umask { sub umask_prepare { my ($self) = @_; my $perm = _git_config_perm($self); - my $umask = _umask_for($perm); - $self->{umask} = $umask; + _umask_for($perm); +} + +sub cleanup ($) { + delete @{$_[0]}{qw(over mm git search)}; +} + +# warnings to ignore when handling spam mailboxes and maybe other places +sub warn_ignore { + my $s = "@_"; + # Email::Address::XS warnings + $s =~ /^Argument contains empty address at / + || $s =~ /^Element at index [0-9]+ contains / + # PublicInbox::MsgTime + || $s =~ /^bogus TZ offset: .+?, ignoring and assuming \+0000/ + || $s =~ /^bad Date: .+? in / + # Encode::Unicode::UTF7 + || $s =~ /^Bad UTF7 data escape at / +} + +# this expects to be RHS in this assignment: "local $SIG{__WARN__} = ..." +sub warn_ignore_cb { + my $cb = $SIG{__WARN__} // \&CORE::warn; + sub { + return if warn_ignore(@_); + $cb->(@_); + } +} + +# v2+ only, XXX: maybe we can just rely on ->max_git_epoch and remove +sub git_dir_latest { + my ($self, $max) = @_; + defined($$max = $self->max_git_epoch) ? + "$self->{inboxdir}/git/$$max.git" : undef; } 1;