import: (v2): write deletes to a separate '_' subdirectory

[public-inbox.git] / lib / PublicInbox / Import.pm

diff --git a/lib/PublicInbox/Import.pm b/lib/PublicInbox/Import.pm
index 7ba166833841ae347106081e4ee76862e7dc21a1..6a640e2360674fce2491f76b21c0ea6ec39c735d 100644 (file)
--- a/lib/PublicInbox/Import.pm
+++ b/lib/PublicInbox/Import.pm
@@ -11,7 +11,6 @@ use Fcntl qw(:flock :DEFAULT);
 use PublicInbox::Spawn qw(spawn);
 use PublicInbox::MID qw(mid_mime mid2path);
 use PublicInbox::Address;
-use PublicInbox::ContentId qw(content_id);
 use PublicInbox::MsgTime qw(msg_timestamp);
 
 sub new {
@@ -122,7 +121,7 @@ sub check_remove_v1 {
        $n = read($r, my $lf, 1);
        defined($n) or die "read final byte of cat-blob failed: $!";
        die "bad read on final byte: <$lf>" if $lf ne "\n";
-       my $cur = PublicInbox::MIME->new($buf);
+       my $cur = PublicInbox::MIME->new(\$buf);
        my $cur_s = $cur->header('Subject');
        $cur_s = '' unless defined $cur_s;
        my $cur_m = $mime->header('Subject');
@@ -133,7 +132,6 @@ sub check_remove_v1 {
        (undef, $cur);
 }
 
-# used for v2 (maybe)
 sub checkpoint {
        my ($self) = @_;
        return unless $self->{pid};
@@ -141,6 +139,15 @@ sub checkpoint {
        undef;
 }
 
+sub progress {
+       my ($self, $msg) = @_;
+       return unless $self->{pid};
+       print { $self->{out} } "progress $msg\n" or wfail;
+       $self->{in}->getline eq "progress $msg\n" or die
+               "progress $msg not received\n";
+       undef;
+}
+
 # used for v2
 sub get_mark {
        my ($self, $mark) = @_;
@@ -155,7 +162,6 @@ sub get_mark {
 # ('MISMATCH', Email::MIME) on mismatch
 # (:MARK, Email::MIME) on success
 #
-# For v2 inboxes, the content_id is returned instead of the msg
 # v2 callers should check with Xapian before calling this as
 # it is not idempotent.
 sub remove {
@@ -171,10 +177,17 @@ sub remove {
                ($err, $cur) = check_remove_v1($r, $w, $tip, $path, $mime);
                return ($err, $cur) if $err;
        } else {
-               $cur = content_id($mime);
-               my $len = length($cur);
+               my $sref;
+               if (ref($mime) eq 'SCALAR') { # optimization used by V2Writable
+                       $sref = $mime;
+               } else { # XXX should not be necessary:
+                       my $str = $mime->as_string;
+                       $sref = \$str;
+               }
+               my $len = length($$sref);
                $blob = $self->{mark}++;
-               print $w "blob\nmark :$blob\ndata $len\n$cur\n" or wfail;
+               print $w "blob\nmark :$blob\ndata $len\n",
+                       $$sref, "\n" or wfail;
        }
 
        my $ref = $self->{ref};
@@ -195,7 +208,7 @@ sub remove {
        if (defined $path) {
                print $w "D $path\n\n" or wfail;
        } else {
-               print $w "M 100644 :$blob d\n\n" or wfail;
+               print $w "M 100644 :$blob _/D\n\n" or wfail;
        }
        $self->{nchg}++;
        (($self->{tip} = ":$commit"), $cur);
@@ -208,15 +221,50 @@ sub parse_date ($) {
        "$ts $zone";
 }
 
-# returns undef on duplicate
-# returns the :MARK of the most recent commit
-sub add {
-       my ($self, $mime, $check_cb) = @_; # mime = Email::MIME
+sub extract_author_info ($) {
+       my ($mime) = @_;
 
+       my $sender = '';
        my $from = $mime->header('From');
        my ($email) = PublicInbox::Address::emails($from);
        my ($name) = PublicInbox::Address::names($from);
+       if (!defined($name) || !defined($email)) {
+               $sender = $mime->header('Sender');
+               if (!defined($name)) {
+                       ($name) = PublicInbox::Address::names($sender);
+               }
+               if (!defined($email)) {
+                       ($email) = PublicInbox::Address::emails($sender);
+               }
+       }
+       if (defined $email) {
+               # quiet down wide character warnings with utf8::encode
+               utf8::encode($email);
+       } else {
+               $email = '';
+               warn "no email in From: $from or Sender: $sender\n";
+       }
+
+       # git gets confused with:
+       #  "'A U Thor <u@example.com>' via foo" <foo@example.com>
+       # ref:
+       # <CAD0k6qSUYANxbjjbE4jTW4EeVwOYgBD=bXkSu=akiYC_CB7Ffw@mail.gmail.com>
+       if (defined $name) {
+               $name =~ tr/<>//d;
+               utf8::encode($name);
+       } else {
+               $name = '';
+               warn "no name in From: $from or Sender: $sender\n";
+       }
+       ($name, $email);
+}
+
+# returns undef on duplicate
+# returns the :MARK of the most recent commit
+sub add {
+       my ($self, $mime, $check_cb) = @_; # mime = Email::MIME
 
+       my ($name, $email) = extract_author_info($mime);
        my $date_raw = parse_date($mime);
        my $subject = $mime->header('Subject');
        $subject = '(no subject)' unless defined $subject;
@@ -263,25 +311,6 @@ sub add {
                print $w "reset $ref\n" or wfail;
        }
 
-       # quiet down wide character warnings with utf8::encode
-       if (defined $email) {
-               utf8::encode($email);
-       } else {
-               $email = '';
-               warn "no email in From: $from\n";
-       }
-
-       # git gets confused with:
-       #  "'A U Thor <u@example.com>' via foo" <foo@example.com>
-       # ref:
-       # <CAD0k6qSUYANxbjjbE4jTW4EeVwOYgBD=bXkSu=akiYC_CB7Ffw@mail.gmail.com>
-       if (defined $name) {
-               $name =~ tr/<>//d;
-               utf8::encode($name);
-       } else {
-               $name = '';
-               warn "no name in From: $from\n";
-       }
        utf8::encode($subject);
        print $w "commit $ref\nmark :$commit\n",
                "author $name <$email> $date_raw\n",