update copyrights for 2021

[public-inbox.git] / lib / PublicInbox / Import.pm

diff --git a/lib/PublicInbox/Import.pm b/lib/PublicInbox/Import.pm
index 1a226cc78dd4e45a429c8f026137251373ff5a8c..47a529ff6767fee583af0138a907a0b4ca8343d5 100644 (file)
--- a/lib/PublicInbox/Import.pm
+++ b/lib/PublicInbox/Import.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2016-2020 all contributors <meta@public-inbox.org>
+# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 #
 # git fast-import-based ssoma-mda MDA replacement
@@ -9,7 +9,7 @@ package PublicInbox::Import;
 use strict;
 use parent qw(PublicInbox::Lock);
 use v5.10.1;
-use PublicInbox::Spawn qw(spawn popen_rd);
+use PublicInbox::Spawn qw(run_die popen_rd);
 use PublicInbox::MID qw(mids mid2path);
 use PublicInbox::Address;
 use PublicInbox::Smsg;
@@ -19,13 +19,21 @@ use PublicInbox::MDA;
 use PublicInbox::Eml;
 use POSIX qw(strftime);
 
+sub default_branch () {
+       state $default_branch = do {
+               my $r = popen_rd([qw(git config --global init.defaultBranch)]);
+               chomp(my $h = <$r> // '');
+               $h eq '' ? 'refs/heads/master' : $h;
+       }
+}
+
 sub new {
        # we can't change arg order, this is documented in POD
        # and external projects may rely on it:
        my ($class, $git, $name, $email, $ibx) = @_;
-       my $ref = 'refs/heads/master';
+       my $ref;
        if ($ibx) {
-               $ref = $ibx->{ref_head} // 'refs/heads/master';
+               $ref = $ibx->{ref_head};
                $name //= $ibx->{name};
                $email //= $ibx->{-primary_address};
                $git //= $ibx->git;
@@ -34,7 +42,7 @@ sub new {
                git => $git,
                ident => "$name <$email>",
                mark => 1,
-               ref => $ref,
+               ref => $ref // default_branch,
                ibx => $ibx,
                path_type => '2/38', # or 'v2'
                lock_path => "$git->{git_dir}/ssoma.lock", # v2 changes this
@@ -48,7 +56,7 @@ sub gfi_start {
 
        return ($self->{in}, $self->{out}) if $self->{pid};
 
-       my (@ret, $out_r, $out_w);
+       my ($in_r, $pid, $out_r, $out_w);
        pipe($out_r, $out_w) or die "pipe failed: $!";
 
        $self->lock_acquire;
@@ -56,27 +64,28 @@ sub gfi_start {
                my ($git, $ref) = @$self{qw(git ref)};
                local $/ = "\n";
                chomp($self->{tip} = $git->qx(qw(rev-parse --revs-only), $ref));
+               die "fatal: rev-parse --revs-only $ref: \$?=$?" if $?;
                if ($self->{path_type} ne '2/38' && $self->{tip}) {
                        local $/ = "\0";
                        my @t = $git->qx(qw(ls-tree -r -z --name-only), $ref);
+                       die "fatal: ls-tree -r -z --name-only $ref: \$?=$?" if $?;
                        chomp @t;
                        $self->{-tree} = { map { $_ => 1 } @t };
                }
                my @cmd = ('git', "--git-dir=$git->{git_dir}",
                        qw(fast-import --quiet --done --date-format=raw));
-               my ($in_r, $pid) = popen_rd(\@cmd, undef, { 0 => $out_r });
+               ($in_r, $pid) = popen_rd(\@cmd, undef, { 0 => $out_r });
                $out_w->autoflush(1);
                $self->{in} = $in_r;
                $self->{out} = $out_w;
                $self->{pid} = $pid;
                $self->{nchg} = 0;
-               @ret = ($in_r, $out_w);
        };
        if ($@) {
                $self->lock_release;
                die $@;
        }
-       @ret;
+       ($in_r, $out_w);
 }
 
 sub wfail () { die "write to fast-import failed: $!" }
@@ -328,11 +337,13 @@ sub extract_cmt_info ($;$) {
 }
 
 # kill potentially confusing/misleading headers
+our @UNWANTED_HEADERS = (qw(Bytes Lines Content-Length),
+                       qw(Status X-Status));
 sub drop_unwanted_headers ($) {
-       my ($mime) = @_;
-
-       $mime->header_set($_) for qw(Bytes Lines Content-Length Status);
-       $mime->header_set($_) for @PublicInbox::MDA::BAD_HEADERS;
+       my ($eml) = @_;
+       for (@UNWANTED_HEADERS, @PublicInbox::MDA::BAD_HEADERS) {
+               $eml->header_set($_);
+       }
 }
 
 # used by V2Writable, too
@@ -403,6 +414,10 @@ sub add {
        if ($smsg) {
                $smsg->{blob} = $self->get_mark(":$blob");
                $smsg->{raw_bytes} = $n;
+               if (my $oidx = delete $smsg->{-oidx}) { # used by LeiStore
+                       return if $oidx->blob_exists($smsg->{blob});
+               }
+               # XXX do we need this? it's in git at this point
                $smsg->{-raw_email} = \$raw_email;
        }
        my $ref = $self->{ref};
@@ -427,14 +442,7 @@ sub add {
        $self->{tip} = ":$commit";
 }
 
-sub run_die ($;$$) {
-       my ($cmd, $env, $rdr) = @_;
-       my $pid = spawn($cmd, $env, $rdr);
-       waitpid($pid, 0) == $pid or die join(' ', @$cmd) .' did not finish';
-       $? == 0 or die join(' ', @$cmd) . " failed: $?\n";
-}
-
-my @INIT_FILES = ('HEAD' => "ref: refs/heads/master\n",
+my @INIT_FILES = ('HEAD' => undef, # filled in at runtime
                'description' => <<EOD,
 Unnamed repository; edit this file 'description' to name the repository.
 EOD
@@ -448,15 +456,18 @@ EOD
 EOC
 
 sub init_bare {
-       my ($dir) = @_; # or self
+       my ($dir, $head) = @_; # or self
        $dir = $dir->{git}->{git_dir} if ref($dir);
        require File::Path;
        File::Path::mkpath([ map { "$dir/$_" } qw(objects/info refs/heads) ]);
-       for (my $i = 0; $i < @INIT_FILES; $i++) {
-               my $f = $dir.'/'.$INIT_FILES[$i++];
+       $INIT_FILES[1] //= 'ref: '.default_branch."\n";
+       my @fn_contents = @INIT_FILES;
+       $fn_contents[1] = "ref: refs/heads/$head\n" if defined $head;
+       while (my ($fn, $contents) = splice(@fn_contents, 0, 2)) {
+               my $f = $dir.'/'.$fn;
                next if -f $f;
                open my $fh, '>', $f or die "open $f: $!";
-               print $fh $INIT_FILES[$i] or die "print $f: $!";
+               print $fh $contents or die "print $f: $!";
                close $fh or die "close $f: $!";
        }
 }