Merge remote-tracking branch 'origin/master' into lorelei

[public-inbox.git] / lib / PublicInbox / Import.pm

diff --git a/lib/PublicInbox/Import.pm b/lib/PublicInbox/Import.pm
index 07a4951871375674b442c1a0c5bd7710013e608f..079afc5f7861320a7bd44e3cb8577b4c86a53456 100644 (file)
--- a/lib/PublicInbox/Import.pm
+++ b/lib/PublicInbox/Import.pm
@@ -48,7 +48,7 @@ sub gfi_start {
 
        return ($self->{in}, $self->{out}) if $self->{pid};
 
-       my (@ret, $out_r, $out_w);
+       my ($in_r, $pid, $out_r, $out_w);
        pipe($out_r, $out_w) or die "pipe failed: $!";
 
        $self->lock_acquire;
@@ -56,27 +56,28 @@ sub gfi_start {
                my ($git, $ref) = @$self{qw(git ref)};
                local $/ = "\n";
                chomp($self->{tip} = $git->qx(qw(rev-parse --revs-only), $ref));
+               die "fatal: rev-parse --revs-only $ref: \$?=$?" if $?;
                if ($self->{path_type} ne '2/38' && $self->{tip}) {
                        local $/ = "\0";
                        my @t = $git->qx(qw(ls-tree -r -z --name-only), $ref);
+                       die "fatal: ls-tree -r -z --name-only $ref: \$?=$?" if $?;
                        chomp @t;
                        $self->{-tree} = { map { $_ => 1 } @t };
                }
                my @cmd = ('git', "--git-dir=$git->{git_dir}",
                        qw(fast-import --quiet --done --date-format=raw));
-               my ($in_r, $pid) = popen_rd(\@cmd, undef, { 0 => $out_r });
+               ($in_r, $pid) = popen_rd(\@cmd, undef, { 0 => $out_r });
                $out_w->autoflush(1);
                $self->{in} = $in_r;
                $self->{out} = $out_w;
                $self->{pid} = $pid;
                $self->{nchg} = 0;
-               @ret = ($in_r, $out_w);
        };
        if ($@) {
                $self->lock_release;
                die $@;
        }
-       @ret;
+       ($in_r, $out_w);
 }
 
 sub wfail () { die "write to fast-import failed: $!" }
@@ -106,7 +107,7 @@ sub _cat_blob ($$$) {
        local $/ = "\n";
        my $info = <$r>;
        defined $info or die "EOF from fast-import / cat-blob: $!";
-       $info =~ /\A[a-f0-9]{40} blob ([0-9]+)\n\z/ or return;
+       $info =~ /\A[a-f0-9]{40,} blob ([0-9]+)\n\z/ or return;
        my $left = $1;
        my $offset = 0;
        my $buf = '';
@@ -137,7 +138,7 @@ sub check_remove_v1 {
        my ($r, $w, $tip, $path, $mime) = @_;
 
        my $info = _check_path($r, $w, $tip, $path) or return ('MISSING',undef);
-       $info =~ m!\A100644 blob ([a-f0-9]{40})\t!s or die "not blob: $info";
+       $info =~ m!\A100644 blob ([a-f0-9]{40,})\t!s or die "not blob: $info";
        my $oid = $1;
        my $msg = _cat_blob($r, $w, $oid) or die "BUG: cat-blob $1 failed";
        my $cur = PublicInbox::Eml->new($msg);
@@ -285,15 +286,14 @@ sub extract_cmt_info ($;$) {
        # $mime is PublicInbox::Eml, but remains Email::MIME-compatible
        $smsg //= bless {}, 'PublicInbox::Smsg';
 
-       my $hdr = $mime->header_obj;
-       $smsg->populate($hdr);
+       $smsg->populate($mime);
 
        my $sender = '';
        my $from = delete($smsg->{From}) // '';
        my ($email) = PublicInbox::Address::emails($from);
        my ($name) = PublicInbox::Address::names($from);
        if (!defined($name) || !defined($email)) {
-               $sender = $hdr->header('Sender') // '';
+               $sender = $mime->header('Sender') // '';
                $name //= (PublicInbox::Address::names($sender))[0];
                $email //= (PublicInbox::Address::emails($sender))[0];
        }
@@ -329,11 +329,13 @@ sub extract_cmt_info ($;$) {
 }
 
 # kill potentially confusing/misleading headers
+our @UNWANTED_HEADERS = (qw(Bytes Lines Content-Length),
+                       qw(Status X-Status));
 sub drop_unwanted_headers ($) {
-       my ($mime) = @_;
-
-       $mime->header_set($_) for qw(Bytes Lines Content-Length Status);
-       $mime->header_set($_) for @PublicInbox::MDA::BAD_HEADERS;
+       my ($eml) = @_;
+       for (@UNWANTED_HEADERS, @PublicInbox::MDA::BAD_HEADERS) {
+               $eml->header_set($_);
+       }
 }
 
 # used by V2Writable, too
@@ -346,13 +348,12 @@ sub append_mid ($$) {
 }
 
 sub v1_mid0 ($) {
-       my ($mime) = @_;
-       my $hdr = $mime->header_obj;
-       my $mids = mids($hdr);
+       my ($eml) = @_;
+       my $mids = mids($eml);
 
        if (!scalar(@$mids)) { # spam often has no Message-ID
-               my $mid0 = digest2mid(content_digest($mime), $hdr);
-               append_mid($hdr, $mid0);
+               my $mid0 = digest2mid(content_digest($eml), $eml);
+               append_mid($eml, $mid0);
                return $mid0;
        }
        $mids->[0];
@@ -405,6 +406,10 @@ sub add {
        if ($smsg) {
                $smsg->{blob} = $self->get_mark(":$blob");
                $smsg->{raw_bytes} = $n;
+               if (my $oidx = delete $smsg->{-oidx}) { # used by LeiStore
+                       return if $oidx->blob_exists($smsg->{blob});
+               }
+               # XXX do we need this? it's in git at this point
                $smsg->{-raw_email} = \$raw_email;
        }
        my $ref = $self->{ref};
@@ -463,6 +468,9 @@ sub init_bare {
        }
 }
 
+# true if locked and active
+sub active { !!$_[0]->{out} }
+
 sub done {
        my ($self) = @_;
        my $w = delete $self->{out} or return;
@@ -671,8 +679,7 @@ version 1.0
        my $parsed = PublicInbox::Eml->new($message);
        my $ret = $im->add($parsed);
        if (!defined $ret) {
-               warn "duplicate: ",
-                       $parsed->header_obj->header_raw('Message-ID'), "\n";
+               warn "duplicate: ", $parsed->header_raw('Message-ID'), "\n";
        } else {
                print "imported at mark $ret\n";
        }