X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FImport.pm;h=0419217419e65116d57c0f2d0613b74f601e0b9a;hb=refs%2Fheads%2Fmaster;hp=6f387b77a1dc07a26afc4ebea9066fce40ec4303;hpb=9c7737d056f832824812086373a13922dd08a0c5;p=public-inbox.git

diff --git a/lib/PublicInbox/Import.pm b/lib/PublicInbox/Import.pm
index 6f387b77..04192174 100644
--- a/lib/PublicInbox/Import.pm
+++ b/lib/PublicInbox/Import.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2016-2020 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 #
 # git fast-import-based ssoma-mda MDA replacement
@@ -21,9 +21,11 @@ use POSIX qw(strftime);
 
 sub default_branch () {
 	state $default_branch = do {
-		my $r = popen_rd([qw(git config --global init.defaultBranch)]);
+		my $r = popen_rd([qw(git config --global init.defaultBranch)],
+				 { GIT_CONFIG => undef });
 		chomp(my $h = <$r> // '');
-		$h eq '' ? 'refs/heads/master' : $h;
+		close $r;
+		$h eq '' ? 'refs/heads/master' : "refs/heads/$h";
 	}
 }
 
@@ -54,9 +56,9 @@ sub new {
 sub gfi_start {
 	my ($self) = @_;
 
-	return ($self->{in}, $self->{out}) if $self->{pid};
+	return ($self->{in}, $self->{out}) if $self->{in};
 
-	my ($in_r, $pid, $out_r, $out_w);
+	my ($in_r, $out_r, $out_w);
 	pipe($out_r, $out_w) or die "pipe failed: $!";
 
 	$self->lock_acquire;
@@ -66,19 +68,15 @@ sub gfi_start {
 		chomp($self->{tip} = $git->qx(qw(rev-parse --revs-only), $ref));
 		die "fatal: rev-parse --revs-only $ref: \$?=$?" if $?;
 		if ($self->{path_type} ne '2/38' && $self->{tip}) {
-			local $/ = "\0";
-			my @t = $git->qx(qw(ls-tree -r -z --name-only), $ref);
+			my $t = $git->qx(qw(ls-tree -r -z --name-only), $ref);
 			die "fatal: ls-tree -r -z --name-only $ref: \$?=$?" if $?;
-			chomp @t;
-			$self->{-tree} = { map { $_ => 1 } @t };
+			$self->{-tree} = { map { $_ => 1 } split(/\0/, $t) };
 		}
-		my @cmd = ('git', "--git-dir=$git->{git_dir}",
-			qw(fast-import --quiet --done --date-format=raw));
-		($in_r, $pid) = popen_rd(\@cmd, undef, { 0 => $out_r });
+		$in_r = $self->{in} = $git->popen(qw(fast-import
+					--quiet --done --date-format=raw),
+					undef, { 0 => $out_r });
 		$out_w->autoflush(1);
-		$self->{in} = $in_r;
 		$self->{out} = $out_w;
-		$self->{pid} = $pid;
 		$self->{nchg} = 0;
 	};
 	if ($@) {
@@ -105,7 +103,7 @@ sub _check_path ($$$$) {
 	return if $tip eq '';
 	print $w "ls $tip $path\n" or wfail;
 	local $/ = "\n";
-	defined(my $info = <$r>) or die "EOF from fast-import: $!";
+	my $info = <$r> // die "EOF from fast-import: $!";
 	$info =~ /\Amissing / ? undef : $info;
 }
 
@@ -113,22 +111,21 @@ sub _cat_blob ($$$) {
 	my ($r, $w, $oid) = @_;
 	print $w "cat-blob $oid\n" or wfail;
 	local $/ = "\n";
-	my $info = <$r>;
-	defined $info or die "EOF from fast-import / cat-blob: $!";
+	my $info = <$r> // die "EOF from fast-import / cat-blob: $!";
 	$info =~ /\A[a-f0-9]{40,} blob ([0-9]+)\n\z/ or return;
 	my $left = $1;
 	my $offset = 0;
 	my $buf = '';
 	my $n;
 	while ($left > 0) {
-		$n = read($r, $buf, $left, $offset);
-		defined($n) or die "read cat-blob failed: $!";
+		$n = read($r, $buf, $left, $offset) //
+			die "read cat-blob failed: $!";
 		$n == 0 and die 'fast-export (cat-blob) died';
 		$left -= $n;
 		$offset += $n;
 	}
-	$n = read($r, my $lf, 1);
-	defined($n) or die "read final byte of cat-blob failed: $!";
+	$n = read($r, my $lf, 1) //
+		die "read final byte of cat-blob failed: $!";
 	die "bad read on final byte: <$lf>" if $lf ne "\n";
 
 	# fixup some bugginess in old versions:
@@ -150,10 +147,8 @@ sub check_remove_v1 {
 	my $oid = $1;
 	my $msg = _cat_blob($r, $w, $oid) or die "BUG: cat-blob $1 failed";
 	my $cur = PublicInbox::Eml->new($msg);
-	my $cur_s = $cur->header('Subject');
-	$cur_s = '' unless defined $cur_s;
-	my $cur_m = $mime->header('Subject');
-	$cur_m = '' unless defined $cur_m;
+	my $cur_s = $cur->header('Subject') // '';
+	my $cur_m = $mime->header('Subject') // '';
 	if ($cur_s ne $cur_m || norm_body($cur) ne norm_body($mime)) {
 		return ('MISMATCH', $cur);
 	}
@@ -162,14 +157,14 @@ sub check_remove_v1 {
 
 sub checkpoint {
 	my ($self) = @_;
-	return unless $self->{pid};
+	return unless $self->{in};
 	print { $self->{out} } "checkpoint\n" or wfail;
 	undef;
 }
 
 sub progress {
 	my ($self, $msg) = @_;
-	return unless $self->{pid};
+	return unless $self->{in};
 	print { $self->{out} } "progress $msg\n" or wfail;
 	readline($self->{in}) eq "progress $msg\n" or die
 		"progress $msg not received\n";
@@ -187,8 +182,8 @@ sub _update_git_info ($$) {
 		my $env = { GIT_INDEX_FILE => $index };
 		run_die([@cmd, qw(read-tree -m -v -i), $self->{ref}], $env);
 	}
-	eval { run_die([@cmd, 'update-server-info']) };
 	my $ibx = $self->{ibx};
+	eval { run_die([@cmd, 'update-server-info']) } if $ibx;
 	if ($ibx && $ibx->version == 1 && -d "$ibx->{inboxdir}/public-inbox" &&
 				eval { require PublicInbox::SearchIdx }) {
 		eval {
@@ -197,7 +192,10 @@ sub _update_git_info ($$) {
 		};
 		warn "$ibx->{inboxdir} index failed: $@\n" if $@;
 	}
-	eval { run_die([@cmd, qw(gc --auto)]) } if $do_gc;
+	if ($do_gc) {
+		my @quiet = (-t STDERR ? () : '-q');
+		eval { run_die([@cmd, qw(gc --auto), @quiet]) }
+	}
 }
 
 sub barrier {
@@ -218,10 +216,10 @@ sub barrier {
 # used for v2
 sub get_mark {
 	my ($self, $mark) = @_;
-	die "not active\n" unless $self->{pid};
+	die "not active\n" unless $self->{in};
 	my ($r, $w) = $self->gfi_start;
 	print $w "get-mark $mark\n" or wfail;
-	defined(my $oid = <$r>) or die "get-mark failed, need git 2.6.0+\n";
+	my $oid = <$r> // die "get-mark failed, need git 2.6.0+\n";
 	chomp($oid);
 	$oid;
 }
@@ -413,12 +411,22 @@ sub add {
 	# v2: we need this for Xapian
 	if ($smsg) {
 		$smsg->{blob} = $self->get_mark(":$blob");
-		$smsg->{raw_bytes} = $n;
+		$smsg->set_bytes($raw_email, $n);
 		if (my $oidx = delete $smsg->{-oidx}) { # used by LeiStore
-			return if $oidx->blob_exists($smsg->{blob});
+			my $eidx_git = delete $smsg->{-eidx_git};
+
+			# we need this sharedkv to dedupe blobs added in the
+			# same fast-import transaction
+			my $u = $self->{uniq_skv} //= do {
+				require PublicInbox::SharedKV;
+				my $x = PublicInbox::SharedKV->new;
+				$x->dbh;
+				$x;
+			};
+			return if !$u->set_maybe($smsg->oidbin, 1);
+			return if (!$oidx->vivify_xvmd($smsg) &&
+					$eidx_git->check($smsg->{blob}));
 		}
-		# XXX do we need this? it's in git at this point
-		$smsg->{-raw_email} = \$raw_email;
 	}
 	my $ref = $self->{ref};
 	my $commit = $self->{mark}++;
@@ -443,9 +451,6 @@ sub add {
 }
 
 my @INIT_FILES = ('HEAD' => undef, # filled in at runtime
-		'description' => <<EOD,
-Unnamed repository; edit this file 'description' to name the repository.
-EOD
 		'config' => <<EOC);
 [core]
 	repositoryFormatVersion = 0
@@ -481,10 +486,7 @@ sub done {
 	eval {
 		my $r = delete $self->{in} or die 'BUG: missing {in} when done';
 		print $w "done\n" or wfail;
-		my $pid = delete $self->{pid} or
-				die 'BUG: missing {pid} when done';
-		waitpid($pid, 0) == $pid or die 'fast-import did not finish';
-		$? == 0 or die "fast-import failed: $?";
+		close $r or die "fast-import failed: $?"; # ProcessPipe::CLOSE
 	};
 	my $wait_err = $@;
 	my $nchg = delete $self->{nchg};
@@ -505,8 +507,8 @@ sub atfork_child {
 	}
 }
 
-sub digest2mid ($$) {
-	my ($dig, $hdr) = @_;
+sub digest2mid ($$;$) {
+	my ($dig, $hdr, $fallback_time) = @_;
 	my $b64 = $dig->clone->b64digest;
 	# Make our own URLs nicer:
 	# See "Base 64 Encoding with URL and Filename Safe Alphabet" in RFC4648
@@ -515,7 +517,7 @@ sub digest2mid ($$) {
 	# Add a date prefix to prevent a leading '-' in case that trips
 	# up some tools (e.g. if a Message-ID were a expected as a
 	# command-line arg)
-	my $dt = msg_datestamp($hdr);
+	my $dt = msg_datestamp($hdr, $fallback_time);
 	$dt = POSIX::strftime('%Y%m%d%H%M%S', gmtime($dt));
 	"$dt.$b64" . '@z';
 }