import: rewrite less history during purge

[public-inbox.git] / lib / PublicInbox / V2Writable.pm

diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index 8e3122abba155e6cd20b7bcd084b4b5665b0053d..b6532ac5c9476540017f9ee9757cdc7c533cfb46 100644 (file)
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -224,11 +224,13 @@ sub purge_oids {
        my ($self, $purge) = @_; # $purge = { $object_id => 1, ... }
        $self->done;
        my $pfx = "$self->{-inbox}->{mainrepo}/git";
+       my $purges = [];
        foreach my $i (0..$self->{max_git}) {
                my $git = PublicInbox::Git->new("$pfx/$i.git");
                my $im = $self->import_init($git, 0);
-               $im->purge_oids($purge);
+               $purges->[$i] = $im->purge_oids($purge);
        }
+       $purges;
 }
 
 sub remove_internal {
@@ -285,7 +287,7 @@ sub remove_internal {
                $self->barrier;
        }
        if ($purge && scalar keys %$purge) {
-               purge_oids($self, $purge);
+               return purge_oids($self, $purge);
        }
        $removed;
 }
@@ -303,24 +305,39 @@ sub purge {
 
 sub done {
        my ($self) = @_;
-       my $locked = defined $self->{idx_parts};
        my $im = delete $self->{im};
        $im->done if $im; # PublicInbox::Import::done
-       $self->searchidx_checkpoint(0);
-       $self->lock_release if $locked;
+
+       if (my $mm = delete $self->{mm}) {
+               $mm->{dbh}->commit;
+       }
+
+       # order matters, we can only close {over} after all partitions
+       # are done because the partitions also write to {over}
+       my $parts = delete $self->{idx_parts};
+       if ($parts) {
+               $_->remote_commit for @$parts;
+               $_->remote_close for @$parts;
+       }
+
+       my $over = $self->{over};
+       $over->remote_commit;
+       $over->remote_close;
+       $self->{transact_bytes} = 0;
+       $self->lock_release if $parts;
 }
 
 sub checkpoint {
        my ($self) = @_;
        my $im = $self->{im};
        $im->checkpoint if $im; # PublicInbox::Import::checkpoint
-       $self->searchidx_checkpoint(1);
+       $self->barrier(1);
 }
 
 # issue a write barrier to ensure all data is visible to other processes
 # and read-only ops.  Order of data importance is: git > SQLite > Xapian
 sub barrier {
-       my ($self) = @_;
+       my ($self, $fsync) = @_;
 
        if (my $im = $self->{im}) {
                $im->barrier;
@@ -339,42 +356,13 @@ sub barrier {
                $_->remote_barrier foreach @$parts;
 
                $over->barrier_wait; # wait for each Xapian partition
+               $over->commit_fsync if $fsync;
 
                $dbh->begin_work;
        }
        $self->{transact_bytes} = 0;
 }
 
-sub searchidx_checkpoint {
-       my ($self, $more) = @_;
-
-       # order matters, we can only close {over} after all partitions
-       # are done because the partitions also write to {over}
-       if (my $parts = $self->{idx_parts}) {
-               foreach my $idx (@$parts) {
-                       $idx->remote_commit; # propagates commit to over
-                       $idx->remote_close unless $more;
-               }
-               delete $self->{idx_parts} unless $more;
-       }
-
-       if (my $mm = $self->{mm}) {
-               my $dbh = $mm->{dbh};
-               $dbh->commit;
-               if ($more) {
-                       $dbh->begin_work;
-               } else {
-                       delete $self->{mm};
-               }
-       }
-       my $over = $self->{over};
-       $over->remote_commit;
-       if (!$more) {
-               $over->remote_close;
-       }
-       $self->{transact_bytes} = 0;
-}
-
 sub git_init {
        my ($self, $new) = @_;
        my $pfx = "$self->{-inbox}->{mainrepo}/git";
@@ -435,7 +423,7 @@ sub importer {
                } else {
                        $self->{im} = undef;
                        $im->done;
-                       $self->searchidx_checkpoint(1);
+                       $self->barrier(1);
                        $im = undef;
                        my $git_dir = $self->git_init(++$self->{max_git});
                        my $git = PublicInbox::Git->new($git_dir);