Merge branch 'eb/index-incremental'

Incremental indexing fixes from Eric W. Biederman.

These prevents the highest message number in msgmap from
being reassigned after deletes in rare cases and ensures
messages are deleted from msgmap in v2.

* eb/index-incremental:
  V2Writeable.pm: In unindex_oid delete the message from msgmap
  V2Writeable.pm: Ensure that a found message number is in the msgmap
  SearchIdx,V2Writeable: Update num_highwater on optimized deletes
  t/v[12]reindex.t: Verify the num highwater is as expected
  t/v[12]reindex.t Verify num_highwater
  Msgmap.pm: Track the largest value of num ever assigned
  SearchIdx.pm: Always assign numbers backwards during incremental indexing
  t/v[12]reindex.t: Test incremental indexing works
  t/v[12]reindex.t: Test that the resulting msgmap is as expected
  t/v[12]reindex.t: Place expected second in Xapian tests
  t/v2reindex.t: Isolate the test cases more
  t/v1reindex.t: Isolate the test cases
  Import.pm: Don't assume {in} and {out} always exist

diff --combined lib/PublicInbox/SearchIdx.pm
index 29868d991bc25ec03c00572ba21d626413fc62b6,54f82aa8ee4ec0df2b5a6684dc3a83153d66128c..ca832ad39d9d2d8520f673d7311127ac66008c3a
--- 1/lib/PublicInbox/SearchIdx.pm
--- 2/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@@ -321,7 -321,8 +321,7 @@@ sub add_xapian ($$$$$) 
                defined $s or return;
  
                my (@orig, @quot);
 -              my $body = $part->body;
 -              my @lines = split(/\n/, $body);
 +              my @lines = split(/\n/, $s);
                while (defined(my $l = shift @lines)) {
                        if ($l =~ /^>/) {
                                $self->index_body(\@orig, $doc) if @orig;
@@@ -582,7 -583,8 +582,8 @@@ sub read_log 
                        my $blob = $1;
                        if (delete $D{$blob}) {
                                if (defined $self->{regen_down}) {
-                                       $self->{regen_down}--;
+                                       my $num = $self->{regen_down}--;
+                                       $self->{mm}->num_highwater($num);
                                }
                                next;
                        }
@@@ -618,23 -620,28 +619,28 @@@ sub _git_log 
        my ($self, $range) = @_;
        my $git = $self->{git};
  
+       # Count the new files so they can be added newest to oldest
+       # and still have numbers increasing from oldest to newest
+       my $fcount = 0;
+       # can't use 'rev-list --count' if we use --diff-filter
+       my $fh = $git->popen(qw(log --pretty=tformat:%h
+                            --no-notes --no-color --no-renames
+                            --diff-filter=AM), $range);
+       ++$fcount while <$fh>;
+       my $high = $self->{mm}->num_highwater;
+ 
        if (index($range, '..') < 0) {
-               my $regen_max = 0;
-               # can't use 'rev-list --count' if we use --diff-filter
-               my $fh = $git->popen(qw(log --pretty=tformat:%h
-                               --no-notes --no-color --no-renames
-                               --diff-filter=AM), $range);
-               ++$regen_max while <$fh>;
-               my (undef, $max) = $self->{mm}->minmax;
- 
-               if ($max && $max == $regen_max) {
+               if ($high && $high == $fcount) {
                        # fix up old bugs in full indexes which caused messages to
                        # not appear in Msgmap
-                       $self->{regen_up} = $max;
+                       $self->{regen_up} = $high;
                } else {
                        # normal regen is for for fresh data
-                       $self->{regen_down} = $regen_max;
+                       $self->{regen_down} = $fcount;
                }
+       } else {
+               # Give oldest messages the smallest numbers
+               $self->{regen_down} = $high + $fcount;
        }
  
        $git->popen(qw/log --no-notes --no-color --no-renames