Merge remote-tracking branch 'origin/purge'

[public-inbox.git] / lib / PublicInbox / V2Writable.pm

diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index c450980c8f51dbb86357602010fa5b1eab455d27..1f17fe212943a51af9b89071f28b07cd7ef74580 100644 (file)
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -2,6 +2,7 @@
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # This interface wraps and mimics PublicInbox::Import
+# Used to write to V2 inboxes (see L<public-inbox-v2-format(5)>).
 package PublicInbox::V2Writable;
 use strict;
 use warnings;
@@ -83,11 +84,14 @@ sub new {
 }
 
 sub init_inbox {
-       my ($self, $parallel) = @_;
+       my ($self, $parallel, $skip) = @_;
        $self->{parallel} = $parallel;
        $self->idx_init;
        my $epoch_max = -1;
        git_dir_latest($self, \$epoch_max);
+       if (defined $skip && $epoch_max == -1) {
+               $epoch_max = $skip;
+       }
        $self->git_init($epoch_max >= 0 ? $epoch_max : 0);
        $self->done;
 }
@@ -160,6 +164,19 @@ sub num_for {
                        return if $existing;
                }
 
+               # AltId may pre-populate article numbers (e.g. X-Mail-Count
+               # or NNTP article number), use that article number if it's
+               # not in Over.
+               my $altid = $self->{-inbox}->{altid};
+               if ($altid && grep(/:file=msgmap\.sqlite3\z/, @$altid)) {
+                       my $num = $self->{mm}->num_for($mid);
+
+                       if (defined $num && !$self->{over}->get_art($num)) {
+                               $$mid0 = $mid;
+                               return $num;
+                       }
+               }
+
                # very unlikely:
                warn "<$mid> reused for mismatched content\n";
 
@@ -220,11 +237,19 @@ sub idx_init {
        # frequently activated.
        delete $ibx->{$_} foreach (qw(git mm search));
 
-       if ($self->{parallel}) {
-               pipe(my ($r, $w)) or die "pipe failed: $!";
-               $self->{bnote} = [ $r, $w ];
-               $w->autoflush(1);
-       }
+       my $indexlevel = $ibx->{indexlevel};
+       if ($indexlevel && $indexlevel eq 'basic') {
+               $self->{parallel} = 0;
+       }
+
+       if ($self->{parallel}) {
+               pipe(my ($r, $w)) or die "pipe failed: $!";
+               # pipe for barrier notifications doesn't need to be big,
+               # 1031: F_SETPIPE_SZ
+               fcntl($w, 1031, 4096) if $^O eq 'linux';
+               $self->{bnote} = [ $r, $w ];
+               $w->autoflush(1);
+       }
 
        my $over = $self->{over};
        $ibx->umask_prepare;
@@ -260,10 +285,19 @@ sub purge_oids {
        $self->done;
        my $pfx = "$self->{-inbox}->{mainrepo}/git";
        my $purges = [];
-       foreach my $i (0..$self->{epoch_max}) {
-               my $git = PublicInbox::Git->new("$pfx/$i.git");
+       my $max = $self->{epoch_max};
+
+       unless (defined($max)) {
+               defined(my $latest = git_dir_latest($self, \$max)) or return;
+               $self->{epoch_max} = $max;
+       }
+       foreach my $i (0..$max) {
+               my $git_dir = "$pfx/$i.git";
+               -d $git_dir or next;
+               my $git = PublicInbox::Git->new($git_dir);
                my $im = $self->import_init($git, 0, 1);
                $purges->[$i] = $im->purge_oids($purge);
+               $im->done;
        }
        $purges;
 }
@@ -365,7 +399,7 @@ sub purge {
        my ($self, $mime) = @_;
        my $purges = $self->{-inbox}->with_umask(sub {
                remove_internal($self, $mime, undef, {});
-       });
+       }) or return;
        $self->idx_init if @$purges; # ->done is called on purges
        for my $i (0..$#$purges) {
                defined(my $cmt = $purges->[$i]) or next;
@@ -472,6 +506,7 @@ sub done {
        delete $self->{bnote};
        $self->{transact_bytes} = 0;
        $self->lock_release if $parts;
+       $self->{-inbox}->git->cleanup;
 }
 
 sub git_init {
@@ -658,7 +693,7 @@ sub mark_deleted {
 }
 
 sub reindex_oid {
-       my ($self, $mm_tmp, $D, $git, $oid, $regen) = @_;
+       my ($self, $mm_tmp, $D, $git, $oid, $regen, $reindex) = @_;
        my $len;
        my $msgref = $git->cat_file($oid, \$len);
        my $mime = PublicInbox::MIME->new($$msgref);
@@ -676,6 +711,7 @@ sub reindex_oid {
                if (defined $n && $n > $num) {
                        $mid0 = $mid;
                        $num = $n;
+                       $self->{mm}->mid_set($num, $mid0);
                }
        }
        if (!defined($mid0) && $regen && !$del) {
@@ -700,7 +736,8 @@ sub reindex_oid {
 
        if (!defined($mid0) || $del) {
                if (!defined($mid0) && $del) { # expected for deletes
-                       $$regen--;
+                       $num = $$regen--;
+                       $self->{mm}->num_highwater($num) unless $reindex;
                        return
                }
 
@@ -767,7 +804,8 @@ sub index_prepare {
                my $git_dir = git_dir_n($self, $i);
                -d $git_dir or next; # missing parts are fine
                my $git = PublicInbox::Git->new($git_dir);
-               chomp(my $tip = $git->qx('rev-parse', $head));
+               chomp(my $tip = $git->qx(qw(rev-parse -q --verify), $head));
+               next if $?; # new repo
                my $range;
                if (defined(my $cur = $ranges->[$i])) {
                        $range = "$cur..$tip";
@@ -840,7 +878,10 @@ sub unindex_oid {
                        warn "BUG: multiple articles linked to $oid\n",
                                join(',',sort keys %gone), "\n";
                }
-               $self->{unindexed}->{$_}++ foreach keys %gone;
+               foreach my $num (keys %gone) {
+                       $self->{unindexed}->{$_}++;
+                       $self->{mm}->num_delete($num);
+               }
                $self->unindex_oid_remote($oid, $mid);
        }
 }
@@ -877,7 +918,8 @@ sub index_sync {
        return unless defined $latest;
        $self->idx_init; # acquire lock
        my $mm_tmp = $self->{mm}->tmp_clone;
-       my $ranges = $opts->{reindex} ? [] : $self->last_commits($epoch_max);
+       my $reindex = $opts->{reindex};
+       my $ranges = $reindex ? [] : $self->last_commits($epoch_max);
 
        my $high = $self->{mm}->num_highwater();
        my $regen = $self->index_prepare($opts, $epoch_max, $ranges);
@@ -903,7 +945,7 @@ sub index_sync {
                                chomp($cmt = $_);
                        } elsif (/\A:\d{6} 100644 $x40 ($x40) [AM]\tm$/o) {
                                $self->reindex_oid($mm_tmp, $D, $git, $1,
-                                               $regen);
+                                               $regen, $reindex);
                        } elsif (/\A:\d{6} 100644 $x40 ($x40) [AM]\td$/o) {
                                $self->mark_deleted($D, $git, $1);
                        }