import: implement barrier operation for v1 repos

[public-inbox.git] / lib / PublicInbox / Import.pm

diff --git a/lib/PublicInbox/Import.pm b/lib/PublicInbox/Import.pm
index e20c6e031f6fb14a64373637a631f8e1ff9edc93..12df7d598fc71d38088d4aeaebd0ed6d8ed3e37d 100644 (file)
--- a/lib/PublicInbox/Import.pm
+++ b/lib/PublicInbox/Import.pm
@@ -11,7 +11,6 @@ use Fcntl qw(:flock :DEFAULT);
 use PublicInbox::Spawn qw(spawn);
 use PublicInbox::MID qw(mid_mime mid2path);
 use PublicInbox::Address;
-use PublicInbox::ContentId qw(content_id);
 use PublicInbox::MsgTime qw(msg_timestamp);
 
 sub new {
@@ -149,6 +148,42 @@ sub progress {
        undef;
 }
 
+sub _update_git_info ($$) {
+       my ($self, $do_gc) = @_;
+       # for compatibility with existing ssoma installations
+       # we can probably remove this entirely by 2020
+       my $git_dir = $self->{git}->{git_dir};
+       my @cmd = ('git', "--git-dir=$git_dir");
+       my $index = "$git_dir/ssoma.index";
+       if (-e $index && !$ENV{FAST}) {
+               my $env = { GIT_INDEX_FILE => $index };
+               run_die([@cmd, qw(read-tree -m -v -i), $self->{ref}], $env);
+       }
+       run_die([@cmd, 'update-server-info'], undef);
+       ($self->{path_type} eq '2/38') and eval {
+               require PublicInbox::SearchIdx;
+               my $inbox = $self->{inbox} || $git_dir;
+               my $s = PublicInbox::SearchIdx->new($inbox);
+               $s->index_sync({ ref => $self->{ref} });
+       };
+       eval { run_die([@cmd, qw(gc --auto)], undef) } if $do_gc;
+}
+
+sub barrier {
+       my ($self) = @_;
+
+       # For safety, we ensure git checkpoint is complete before because
+       # the data in git is still more important than what is in Xapian
+       # in v2.  Performance may be gained by delaying the ->progress
+       # call but we lose safety
+       if ($self->{nchg}) {
+               $self->checkpoint;
+               $self->progress('checkpoint');
+               _update_git_info($self, 0);
+               $self->{nchg} = 0;
+       }
+}
+
 # used for v2
 sub get_mark {
        my ($self, $mark) = @_;
@@ -163,7 +198,6 @@ sub get_mark {
 # ('MISMATCH', Email::MIME) on mismatch
 # (:MARK, Email::MIME) on success
 #
-# For v2 inboxes, the content_id is returned instead of the msg
 # v2 callers should check with Xapian before calling this as
 # it is not idempotent.
 sub remove {
@@ -179,10 +213,17 @@ sub remove {
                ($err, $cur) = check_remove_v1($r, $w, $tip, $path, $mime);
                return ($err, $cur) if $err;
        } else {
-               $cur = content_id($mime);
-               my $len = length($cur);
+               my $sref;
+               if (ref($mime) eq 'SCALAR') { # optimization used by V2Writable
+                       $sref = $mime;
+               } else { # XXX should not be necessary:
+                       my $str = $mime->as_string;
+                       $sref = \$str;
+               }
+               my $len = length($$sref);
                $blob = $self->{mark}++;
-               print $w "blob\nmark :$blob\ndata $len\n$cur\n" or wfail;
+               print $w "blob\nmark :$blob\ndata $len\n",
+                       $$sref, "\n" or wfail;
        }
 
        my $ref = $self->{ref};
@@ -203,7 +244,7 @@ sub remove {
        if (defined $path) {
                print $w "D $path\n\n" or wfail;
        } else {
-               print $w "M 100644 :$blob d\n\n" or wfail;
+               print $w "M 100644 :$blob _/D\n\n" or wfail;
        }
        $self->{nchg}++;
        (($self->{tip} = ":$commit"), $cur);
@@ -336,28 +377,8 @@ sub done {
        my $pid = delete $self->{pid} or die 'BUG: missing {pid} when done';
        waitpid($pid, 0) == $pid or die 'fast-import did not finish';
        $? == 0 or die "fast-import failed: $?";
-       my $nchg = delete $self->{nchg};
 
-       # for compatibility with existing ssoma installations
-       # we can probably remove this entirely by 2020
-       my $git_dir = $self->{git}->{git_dir};
-       my @cmd = ('git', "--git-dir=$git_dir");
-       my $index = "$git_dir/ssoma.index";
-       if ($nchg && -e $index && !$ENV{FAST}) {
-               my $env = { GIT_INDEX_FILE => $index };
-               run_die([@cmd, qw(read-tree -m -v -i), $self->{ref}], $env);
-       }
-       if ($nchg) {
-               run_die([@cmd, 'update-server-info'], undef);
-               ($self->{path_type} eq '2/38') and eval {
-                       require PublicInbox::SearchIdx;
-                       my $inbox = $self->{inbox} || $git_dir;
-                       my $s = PublicInbox::SearchIdx->new($inbox);
-                       $s->index_sync({ ref => $self->{ref} });
-               };
-
-               eval { run_die([@cmd, qw(gc --auto)], undef) };
-       }
+       _update_git_info($self, 1) if delete $self->{nchg};
 
        $self->{ssoma_lock} or return;
        my $lockfh = delete $self->{lockfh} or die "BUG: not locked: $!";