use both Date: and Received: times

[public-inbox.git] / lib / PublicInbox / Import.pm

diff --git a/lib/PublicInbox/Import.pm b/lib/PublicInbox/Import.pm
index 12df7d598fc71d38088d4aeaebd0ed6d8ed3e37d..d69934b0678aa2e7927b02f218524f4e7f4782c0 100644 (file)
--- a/lib/PublicInbox/Import.pm
+++ b/lib/PublicInbox/Import.pm
@@ -7,11 +7,13 @@
 package PublicInbox::Import;
 use strict;
 use warnings;
-use Fcntl qw(:flock :DEFAULT);
+use base qw(PublicInbox::Lock);
 use PublicInbox::Spawn qw(spawn);
-use PublicInbox::MID qw(mid_mime mid2path);
+use PublicInbox::MID qw(mids mid_mime mid2path);
 use PublicInbox::Address;
-use PublicInbox::MsgTime qw(msg_timestamp);
+use PublicInbox::MsgTime qw(msg_timestamp msg_datestamp);
+use PublicInbox::ContentId qw(content_digest);
+use PublicInbox::MDA;
 
 sub new {
        my ($class, $git, $name, $email, $ibx) = @_;
@@ -28,7 +30,7 @@ sub new {
                ref => $ref,
                inbox => $ibx,
                path_type => '2/38', # or 'v2'
-               ssoma_lock => 1, # disable for v2
+               lock_path => "$git->{git_dir}/ssoma.lock", # v2 changes this
                bytes_added => 0,
        }, $class
 }
@@ -43,20 +45,13 @@ sub gfi_start {
        pipe($in_r, $in_w) or die "pipe failed: $!";
        pipe($out_r, $out_w) or die "pipe failed: $!";
        my $git = $self->{git};
-       my $git_dir = $git->{git_dir};
 
-       my $lockfh;
-       if ($self->{ssoma_lock}) {
-               my $lockpath = "$git_dir/ssoma.lock";
-               sysopen($lockfh, $lockpath, O_WRONLY|O_CREAT) or
-                       die "failed to open lock $lockpath: $!";
-               # wait for other processes to be done
-               flock($lockfh, LOCK_EX) or die "lock failed: $!\n";
-       }
+       $self->lock_acquire;
 
        local $/ = "\n";
        chomp($self->{tip} = $git->qx(qw(rev-parse --revs-only), $self->{ref}));
 
+       my $git_dir = $git->{git_dir};
        my @cmd = ('git', "--git-dir=$git_dir", qw(fast-import
                        --quiet --done --date-format=raw));
        my $rdr = { 0 => fileno($out_r), 1 => fileno($in_w) };
@@ -65,7 +60,6 @@ sub gfi_start {
        $out_w->autoflush(1);
        $self->{in} = $in_r;
        $self->{out} = $out_w;
-       $self->{lockfh} = $lockfh;
        $self->{pid} = $pid;
        $self->{nchg} = 0;
        binmode $out_w, ':raw' or die "binmode :raw failed: $!";
@@ -250,9 +244,8 @@ sub remove {
        (($self->{tip} = ":$commit"), $cur);
 }
 
-sub parse_date ($) {
-       my ($mime) = @_;
-       my ($ts, $zone) = msg_timestamp($mime->header_obj);
+sub git_timestamp {
+       my ($ts, $zone) = @_;
        $ts = 0 if $ts < 0; # git uses unsigned times
        "$ts $zone";
 }
@@ -301,14 +294,23 @@ sub add {
        my ($self, $mime, $check_cb) = @_; # mime = Email::MIME
 
        my ($name, $email) = extract_author_info($mime);
-       my $date_raw = parse_date($mime);
+       my $hdr = $mime->header_obj;
+       my @at = msg_datestamp($hdr);
+       my @ct = msg_timestamp($hdr);
+       my $author_time_raw = git_timestamp(@at);
+       my $commit_time_raw = git_timestamp(@ct);
        my $subject = $mime->header('Subject');
        $subject = '(no subject)' unless defined $subject;
        my $path_type = $self->{path_type};
 
        my $path;
        if ($path_type eq '2/38') {
-               $path = mid2path(mid_mime($mime));
+               my $mids = mids($mime->header_obj);
+               if (!scalar(@$mids)) {
+                       my $dig = content_digest($mime);
+                       @$mids = (digest2mid($dig));
+               }
+               $path = mid2path($mids->[0]);
        } else { # v2 layout, one file:
                $path = 'm';
        }
@@ -321,6 +323,7 @@ sub add {
 
        # kill potentially confusing/misleading headers
        $mime->header_set($_) for qw(bytes lines content-length status);
+       $mime->header_set($_) for @PublicInbox::MDA::BAD_HEADERS;
 
        # spam check:
        if ($check_cb) {
@@ -349,8 +352,8 @@ sub add {
 
        utf8::encode($subject);
        print $w "commit $ref\nmark :$commit\n",
-               "author $name <$email> $date_raw\n",
-               "committer $self->{ident} ", now_raw(), "\n" or wfail;
+               "author $name <$email> $author_time_raw\n",
+               "committer $self->{ident} $commit_time_raw\n" or wfail;
        print $w "data ", (length($subject) + 1), "\n",
                $subject, "\n\n" or wfail;
        if ($tip ne '') {
@@ -380,10 +383,7 @@ sub done {
 
        _update_git_info($self, 1) if delete $self->{nchg};
 
-       $self->{ssoma_lock} or return;
-       my $lockfh = delete $self->{lockfh} or die "BUG: not locked: $!";
-       flock($lockfh, LOCK_UN) or die "unlock failed: $!";
-       close $lockfh or die "close lock failed: $!";
+       $self->lock_release;
 }
 
 sub atfork_child {
@@ -393,6 +393,20 @@ sub atfork_child {
        }
 }
 
+sub digest2mid ($) {
+       my ($dig) = @_;
+       my $b64 = $dig->clone->b64digest;
+       # Make our own URLs nicer:
+       # See "Base 64 Encoding with URL and Filename Safe Alphabet" in RFC4648
+       $b64 =~ tr!+/=!-_!d;
+
+       # We can make this more meaningful with a date prefix or other things,
+       # but this is only needed for crap that fails to generate a Message-ID
+       # or reuses one.  In other words, it's usually spammers who hit this
+       # so they don't deserve nice Message-IDs :P
+       $b64 . '@localhost';
+}
+
 1;
 __END__
 =pod