X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FImport.pm;h=ee5ca2ea11b4f383e514ecdf117ac1dabe2c14c1;hb=94ae705673cb03045a109041eec9a6704b8a735b;hp=af35905be4964b87ad9ea5dcb7eb78b7fe3e9d9a;hpb=cd389aac52936c82f3416b3ceefe21e1250b8a3e;p=public-inbox.git

diff --git a/lib/PublicInbox/Import.pm b/lib/PublicInbox/Import.pm
index af35905b..ee5ca2ea 100644
--- a/lib/PublicInbox/Import.pm
+++ b/lib/PublicInbox/Import.pm
@@ -35,7 +35,7 @@ sub new {
 		ident => "$name <$email>",
 		mark => 1,
 		ref => $ref,
-		-inbox => $ibx,
+		ibx => $ibx,
 		path_type => '2/38', # or 'v2'
 		lock_path => "$git->{git_dir}/ssoma.lock", # v2 changes this
 		bytes_added => 0,
@@ -48,34 +48,35 @@ sub gfi_start {
 
 	return ($self->{in}, $self->{out}) if $self->{pid};
 
-	my ($out_r, $out_w);
+	my (@ret, $out_r, $out_w);
 	pipe($out_r, $out_w) or die "pipe failed: $!";
-	my $git = $self->{git};
 
 	$self->lock_acquire;
-
-	local $/ = "\n";
-	my $ref = $self->{ref};
-	chomp($self->{tip} = $git->qx(qw(rev-parse --revs-only), $ref));
-	if ($self->{path_type} ne '2/38' && $self->{tip}) {
-		local $/ = "\0";
-		my @tree = $git->qx(qw(ls-tree -r -z --name-only), $ref);
-		chomp @tree;
-		$self->{-tree} = { map { $_ => 1 } @tree };
+	eval {
+		my ($git, $ref) = @$self{qw(git ref)};
+		local $/ = "\n";
+		chomp($self->{tip} = $git->qx(qw(rev-parse --revs-only), $ref));
+		if ($self->{path_type} ne '2/38' && $self->{tip}) {
+			local $/ = "\0";
+			my @t = $git->qx(qw(ls-tree -r -z --name-only), $ref);
+			chomp @t;
+			$self->{-tree} = { map { $_ => 1 } @t };
+		}
+		my @cmd = ('git', "--git-dir=$git->{git_dir}",
+			qw(fast-import --quiet --done --date-format=raw));
+		my ($in_r, $pid) = popen_rd(\@cmd, undef, { 0 => $out_r });
+		$out_w->autoflush(1);
+		$self->{in} = $in_r;
+		$self->{out} = $out_w;
+		$self->{pid} = $pid;
+		$self->{nchg} = 0;
+		@ret = ($in_r, $out_w);
+	};
+	if ($@) {
+		$self->lock_release;
+		die $@;
 	}
-
-	my $git_dir = $git->{git_dir};
-	my @cmd = ('git', "--git-dir=$git_dir", qw(fast-import
-			--quiet --done --date-format=raw));
-	my ($in_r, $pid) = popen_rd(\@cmd, undef, { 0 => $out_r });
-	$out_w->autoflush(1);
-	$self->{in} = $in_r;
-	$self->{out} = $out_w;
-	$self->{pid} = $pid;
-	$self->{nchg} = 0;
-	binmode $out_w, ':raw' or die "binmode :raw failed: $!";
-	binmode $in_r, ':raw' or die "binmode :raw failed: $!";
-	($in_r, $out_w);
+	@ret;
 }
 
 sub wfail () { die "write to fast-import failed: $!" }
@@ -177,13 +178,16 @@ sub _update_git_info ($$) {
 		my $env = { GIT_INDEX_FILE => $index };
 		run_die([@cmd, qw(read-tree -m -v -i), $self->{ref}], $env);
 	}
-	run_die([@cmd, 'update-server-info']);
-	my $ibx = $self->{-inbox};
-	($ibx && $self->{path_type} eq '2/38') and eval {
-		require PublicInbox::SearchIdx;
-		my $s = PublicInbox::SearchIdx->new($ibx);
-		$s->index_sync({ ref => $self->{ref} });
-	};
+	eval { run_die([@cmd, 'update-server-info']) };
+	my $ibx = $self->{ibx};
+	if ($ibx && $ibx->version == 1 && -d "$ibx->{inboxdir}/public-inbox" &&
+				eval { require PublicInbox::SearchIdx }) {
+		eval {
+			my $s = PublicInbox::SearchIdx->new($ibx);
+			$s->index_sync({ ref => $self->{ref} });
+		};
+		warn "$ibx->{inboxdir} index failed: $@\n" if $@;
+	}
 	eval { run_die([@cmd, qw(gc --auto)]) } if $do_gc;
 }
 
@@ -281,15 +285,14 @@ sub extract_cmt_info ($;$) {
 	# $mime is PublicInbox::Eml, but remains Email::MIME-compatible
 	$smsg //= bless {}, 'PublicInbox::Smsg';
 
-	my $hdr = $mime->header_obj;
-	$smsg->populate($hdr);
+	$smsg->populate($mime);
 
 	my $sender = '';
 	my $from = delete($smsg->{From}) // '';
 	my ($email) = PublicInbox::Address::emails($from);
 	my ($name) = PublicInbox::Address::names($from);
 	if (!defined($name) || !defined($email)) {
-		$sender = $hdr->header('Sender') // '';
+		$sender = $mime->header('Sender') // '';
 		$name //= (PublicInbox::Address::names($sender))[0];
 		$email //= (PublicInbox::Address::emails($sender))[0];
 	}
@@ -328,7 +331,7 @@ sub extract_cmt_info ($;$) {
 sub drop_unwanted_headers ($) {
 	my ($mime) = @_;
 
-	$mime->header_set($_) for qw(bytes lines content-length status);
+	$mime->header_set($_) for qw(Bytes Lines Content-Length Status);
 	$mime->header_set($_) for @PublicInbox::MDA::BAD_HEADERS;
 }
 
@@ -342,13 +345,12 @@ sub append_mid ($$) {
 }
 
 sub v1_mid0 ($) {
-	my ($mime) = @_;
-	my $hdr = $mime->header_obj;
-	my $mids = mids($hdr);
+	my ($eml) = @_;
+	my $mids = mids($eml);
 
 	if (!scalar(@$mids)) { # spam often has no Message-ID
-		my $mid0 = digest2mid(content_digest($mime), $hdr);
-		append_mid($hdr, $mid0);
+		my $mid0 = digest2mid(content_digest($eml), $eml);
+		append_mid($eml, $mid0);
 		return $mid0;
 	}
 	$mids->[0];
@@ -387,7 +389,7 @@ sub add {
 
 	# spam check:
 	if ($check_cb) {
-		$mime = $check_cb->($mime) or return;
+		$mime = $check_cb->($mime, $self->{ibx}) or return;
 	}
 
 	my $blob = $self->{mark}++;
@@ -459,20 +461,29 @@ sub init_bare {
 	}
 }
 
+# true if locked and active
+sub active { !!$_[0]->{out} }
+
 sub done {
 	my ($self) = @_;
 	my $w = delete $self->{out} or return;
-	my $r = delete $self->{in} or die 'BUG: missing {in} when done';
-	print $w "done\n" or wfail;
-	my $pid = delete $self->{pid} or die 'BUG: missing {pid} when done';
-	waitpid($pid, 0) == $pid or die 'fast-import did not finish';
-	$? == 0 or die "fast-import failed: $?";
-
-	_update_git_info($self, 1) if delete $self->{nchg};
-
-	$self->lock_release;
-
+	eval {
+		my $r = delete $self->{in} or die 'BUG: missing {in} when done';
+		print $w "done\n" or wfail;
+		my $pid = delete $self->{pid} or
+				die 'BUG: missing {pid} when done';
+		waitpid($pid, 0) == $pid or die 'fast-import did not finish';
+		$? == 0 or die "fast-import failed: $?";
+	};
+	my $wait_err = $@;
+	my $nchg = delete $self->{nchg};
+	if ($nchg && !$wait_err) {
+		eval { _update_git_info($self, 1) };
+		warn "E: $self->{git}->{git_dir} update info: $@\n" if $@;
+	}
+	$self->lock_release(!!$nchg);
 	$self->{git}->cleanup;
+	die $wait_err if $wait_err;
 }
 
 sub atfork_child {
@@ -661,8 +672,7 @@ version 1.0
 	my $parsed = PublicInbox::Eml->new($message);
 	my $ret = $im->add($parsed);
 	if (!defined $ret) {
-		warn "duplicate: ",
-			$parsed->header_obj->header_raw('Message-ID'), "\n";
+		warn "duplicate: ", $parsed->header_raw('Message-ID'), "\n";
 	} else {
 		print "imported at mark $ret\n";
 	}