X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FSearchIdx.pm;h=114420e4b82a2b4f71fce48c8538d7ede36e9ea2;hb=e220b8b2ee5cfd458167dc2c6c92726352c4c80e;hp=36f97b36d7bb8170237884bd27742cc6afed0fd1;hpb=0321a1a9e7ae9c9d878d547ee67659ef8aa95689;p=public-inbox.git

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 36f97b36..114420e4 100644
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -12,42 +12,30 @@ use warnings;
 use base qw(PublicInbox::Search PublicInbox::Lock);
 use PublicInbox::MIME;
 use PublicInbox::InboxWritable;
-use PublicInbox::MID qw/mid_clean id_compress mid_mime mids references/;
+use PublicInbox::MID qw/mid_clean id_compress mid_mime mids/;
 use PublicInbox::MsgIter;
 use Carp qw(croak);
 use POSIX qw(strftime);
-require PublicInbox::Git;
+use PublicInbox::OverIdx;
+use PublicInbox::Spawn qw(spawn);
+use PublicInbox::Git qw(git_unquote);
 
 use constant {
-	BATCH_BYTES => 1_000_000,
+	BATCH_BYTES => defined($ENV{XAPIAN_FLUSH_THRESHOLD}) ?
+			0x7fffffff : 1_000_000,
 	DEBUG => !!$ENV{DEBUG},
 };
 
-my %GIT_ESC = (
-	a => "\a",
-	b => "\b",
-	f => "\f",
-	n => "\n",
-	r => "\r",
-	t => "\t",
-	v => "\013",
-);
-
-sub git_unquote ($) {
-	my ($s) = @_;
-	return $s unless ($s =~ /\A"(.*)"\z/);
-	$s = $1;
-	$s =~ s/\\([abfnrtv])/$GIT_ESC{$1}/g;
-	$s =~ s/\\([0-7]{1,3})/chr(oct($1))/ge;
-	$s;
-}
+my $xapianlevels = qr/\A(?:full|medium)\z/;
 
 sub new {
 	my ($class, $ibx, $creat, $part) = @_;
+	my $levels = qr/\A(?:full|medium|basic)\z/;
 	my $mainrepo = $ibx; # for "public-inbox-index" w/o entry in config
 	my $git_dir = $mainrepo;
 	my ($altid, $git);
 	my $version = 1;
+	my $indexlevel = 'full';
 	if (ref $ibx) {
 		$mainrepo = $ibx->{mainrepo};
 		$altid = $ibx->{altid};
@@ -58,27 +46,35 @@ sub new {
 				PublicInbox::AltId->new($ibx, $_);
 			} @$altid ];
 		}
-	} else { # v1
+		if ($ibx->{indexlevel}) {
+			if ($ibx->{indexlevel} =~ $levels) {
+				$indexlevel = $ibx->{indexlevel};
+			} else {
+				die("Invalid indexlevel $ibx->{indexlevel}\n");
+			}
+		}
+	} else { # FIXME: old tests: old tests
 		$ibx = { mainrepo => $git_dir, version => 1 };
 	}
 	$ibx = PublicInbox::InboxWritable->new($ibx);
-	require Search::Xapian::WritableDatabase;
 	my $self = bless {
 		mainrepo => $mainrepo,
 		-inbox => $ibx,
 		git => $ibx->git,
 		-altid => $altid,
 		version => $version,
+		indexlevel => $indexlevel,
 	}, $class;
 	$ibx->umask_prepare;
 	if ($version == 1) {
 		$self->{lock_path} = "$mainrepo/ssoma.lock";
+		my $dir = $self->xdir;
+		$self->{over} = PublicInbox::OverIdx->new("$dir/over.sqlite3");
 	} elsif ($version == 2) {
 		defined $part or die "partition is required for v2\n";
-		# partition is a number or "all"
+		# partition is a number
 		$self->{partition} = $part;
 		$self->{lock_path} = undef;
-		$self->{msgmap_path} = "$mainrepo/msgmap.sqlite3";
 	} else {
 		die "unsupported inbox version=$version\n";
 	}
@@ -86,25 +82,41 @@ sub new {
 	$self;
 }
 
+sub need_xapian ($) { $_[0]->{indexlevel} =~ $xapianlevels }
+
 sub _xdb_release {
 	my ($self) = @_;
-	my $xdb = delete $self->{xdb} or croak 'not acquired';
-	$xdb->close;
+	if (need_xapian($self)) {
+		my $xdb = delete $self->{xdb} or croak 'not acquired';
+		$xdb->close;
+	}
 	$self->lock_release if $self->{creat};
 	undef;
 }
 
 sub _xdb_acquire {
 	my ($self) = @_;
-	croak 'already acquired' if $self->{xdb};
+	my $flag;
 	my $dir = $self->xdir;
-	my $flag = Search::Xapian::DB_OPEN;
+	if (need_xapian($self)) {
+		croak 'already acquired' if $self->{xdb};
+		PublicInbox::Search::load_xapian();
+		require Search::Xapian::WritableDatabase;
+		$flag = $self->{creat} ?
+			Search::Xapian::DB_CREATE_OR_OPEN() :
+			Search::Xapian::DB_OPEN();
+	}
 	if ($self->{creat}) {
 		require File::Path;
 		$self->lock_acquire;
-		File::Path::mkpath($dir);
-		$flag = Search::Xapian::DB_CREATE_OR_OPEN;
+
+		# don't create empty Xapian directories if we don't need Xapian
+		my $is_part = defined($self->{partition});
+		if (!$is_part || ($is_part && need_xapian($self))) {
+			File::Path::mkpath($dir);
+		}
 	}
+	return unless defined $flag;
 	$self->{xdb} = Search::Xapian::WritableDatabase->new($dir, $flag);
 }
 
@@ -114,55 +126,42 @@ sub add_val ($$$) {
 	$doc->add_value($col, $num);
 }
 
-sub add_values ($$) {
-	my ($doc, $values) = @_;
-
-	my $ts = $values->[PublicInbox::Search::TS];
-	add_val($doc, PublicInbox::Search::TS, $ts);
-
-	my $num = $values->[PublicInbox::Search::NUM];
-	defined($num) and add_val($doc, PublicInbox::Search::NUM, $num);
-
-	my $bytes = $values->[PublicInbox::Search::BYTES];
-	defined($bytes) and add_val($doc, PublicInbox::Search::BYTES, $bytes);
+sub index_text ($$$$)
+{
+	my ($self, $field, $n, $text) = @_;
+	my $tg = $self->term_generator;
 
-	my $lines = $values->[PublicInbox::Search::LINES];
-	add_val($doc, PublicInbox::Search::LINES, $lines);
-
-	my $ds = $values->[PublicInbox::Search::DS];
-	add_val($doc, PublicInbox::Search::DS, $ds);
-	my $yyyymmdd = strftime('%Y%m%d', gmtime($ds));
-	add_val($doc, PublicInbox::Search::YYYYMMDD, $yyyymmdd);
+	if ($self->{indexlevel} eq 'full') {
+		$tg->index_text($field, $n, $text);
+		$tg->increase_termpos;
+	} else {
+		$tg->index_text_without_positions($field, $n, $text);
+	}
 }
 
 sub index_users ($$) {
-	my ($tg, $smsg) = @_;
+	my ($self, $smsg) = @_;
 
 	my $from = $smsg->from;
 	my $to = $smsg->to;
 	my $cc = $smsg->cc;
 
-	$tg->index_text($from, 1, 'A'); # A - author
-	$tg->increase_termpos;
-	$tg->index_text($to, 1, 'XTO') if $to ne '';
-	$tg->increase_termpos;
-	$tg->index_text($cc, 1, 'XCC') if $cc ne '';
-	$tg->increase_termpos;
+	$self->index_text($from, 1, 'A'); # A - author
+	$self->index_text($to, 1, 'XTO') if $to ne '';
+	$self->index_text($cc, 1, 'XCC') if $cc ne '';
 }
 
 sub index_diff_inc ($$$$) {
-	my ($tg, $text, $pfx, $xnq) = @_;
+	my ($self, $text, $pfx, $xnq) = @_;
 	if (@$xnq) {
-		$tg->index_text(join("\n", @$xnq), 1, 'XNQ');
-		$tg->increase_termpos;
+		$self->index_text(join("\n", @$xnq), 1, 'XNQ');
 		@$xnq = ();
 	}
-	$tg->index_text($text, 1, $pfx);
-	$tg->increase_termpos;
+	$self->index_text($text, 1, $pfx);
 }
 
 sub index_old_diff_fn {
-	my ($tg, $seen, $fa, $fb, $xnq) = @_;
+	my ($self, $seen, $fa, $fb, $xnq) = @_;
 
 	# no renames or space support for traditional diffs,
 	# find the number of leading common paths to strip:
@@ -173,7 +172,7 @@ sub index_old_diff_fn {
 		$fb = join('/', @fb);
 		if ($fa eq $fb) {
 			unless ($seen->{$fa}++) {
-				index_diff_inc($tg, $fa, 'XDFN', $xnq);
+				$self->index_diff_inc($fa, 'XDFN', $xnq);
 			}
 			return 1;
 		}
@@ -184,22 +183,22 @@ sub index_old_diff_fn {
 }
 
 sub index_diff ($$$) {
-	my ($tg, $lines, $doc) = @_;
+	my ($self, $lines, $doc) = @_;
 	my %seen;
 	my $in_diff;
 	my @xnq;
 	my $xnq = \@xnq;
 	foreach (@$lines) {
 		if ($in_diff && s/^ //) { # diff context
-			index_diff_inc($tg, $_, 'XDFCTX', $xnq);
+			$self->index_diff_inc($_, 'XDFCTX', $xnq);
 		} elsif (/^-- $/) { # email signature begins
 			$in_diff = undef;
 		} elsif (m!^diff --git ("?a/.+) ("?b/.+)\z!) {
 			my ($fa, $fb) = ($1, $2);
 			my $fn = (split('/', git_unquote($fa), 2))[1];
-			$seen{$fn}++ or index_diff_inc($tg, $fn, 'XDFN', $xnq);
+			$seen{$fn}++ or $self->index_diff_inc($fn, 'XDFN', $xnq);
 			$fn = (split('/', git_unquote($fb), 2))[1];
-			$seen{$fn}++ or index_diff_inc($tg, $fn, 'XDFN', $xnq);
+			$seen{$fn}++ or $self->index_diff_inc($fn, 'XDFN', $xnq);
 			$in_diff = 1;
 		# traditional diff:
 		} elsif (m/^diff -(.+) (\S+) (\S+)$/) {
@@ -207,26 +206,26 @@ sub index_diff ($$$) {
 			push @xnq, $_;
 			# only support unified:
 			next unless $opt =~ /[uU]/;
-			$in_diff = index_old_diff_fn($tg, \%seen, $fa, $fb,
+			$in_diff = $self->index_old_diff_fn(\%seen, $fa, $fb,
 							$xnq);
 		} elsif (m!^--- ("?a/.+)!) {
 			my $fn = (split('/', git_unquote($1), 2))[1];
-			$seen{$fn}++ or index_diff_inc($tg, $fn, 'XDFN', $xnq);
+			$seen{$fn}++ or $self->index_diff_inc($fn, 'XDFN', $xnq);
 			$in_diff = 1;
 		} elsif (m!^\+\+\+ ("?b/.+)!)  {
 			my $fn = (split('/', git_unquote($1), 2))[1];
-			$seen{$fn}++ or index_diff_inc($tg, $fn, 'XDFN', $xnq);
+			$seen{$fn}++ or $self->index_diff_inc($fn, 'XDFN', $xnq);
 			$in_diff = 1;
 		} elsif (/^--- (\S+)/) {
 			$in_diff = $1;
 			push @xnq, $_;
 		} elsif (defined $in_diff && /^\+\+\+ (\S+)/) {
-			$in_diff = index_old_diff_fn($tg, \%seen, $in_diff, $1,
+			$in_diff = $self->index_old_diff_fn(\%seen, $in_diff, $1,
 							$xnq);
 		} elsif ($in_diff && s/^\+//) { # diff added
-			index_diff_inc($tg, $_, 'XDFB', $xnq);
+			$self->index_diff_inc($_, 'XDFB', $xnq);
 		} elsif ($in_diff && s/^-//) { # diff removed
-			index_diff_inc($tg, $_, 'XDFA', $xnq);
+			$self->index_diff_inc($_, 'XDFA', $xnq);
 		} elsif (m!^index ([a-f0-9]+)\.\.([a-f0-9]+)!) {
 			my ($ba, $bb) = ($1, $2);
 			index_git_blob_id($doc, 'XDFPRE', $ba);
@@ -236,7 +235,7 @@ sub index_diff ($$$) {
 			# traditional diff w/o -p
 		} elsif (/^@@ (?:\S+) (?:\S+) @@\s*(\S+.*)$/) {
 			# hunk header context
-			index_diff_inc($tg, $1, 'XDFHH', $xnq);
+			$self->index_diff_inc($1, 'XDFHH', $xnq);
 		# ignore the following lines:
 		} elsif (/^(?:dis)similarity index/ ||
 				/^(?:old|new) mode/ ||
@@ -255,125 +254,113 @@ sub index_diff ($$$) {
 		}
 	}
 
-	$tg->index_text(join("\n", @xnq), 1, 'XNQ');
-	$tg->increase_termpos;
+	$self->index_text(join("\n", @xnq), 1, 'XNQ');
 }
 
 sub index_body ($$$) {
-	my ($tg, $lines, $doc) = @_;
+	my ($self, $lines, $doc) = @_;
 	my $txt = join("\n", @$lines);
 	if ($doc) {
 		# does it look like a diff?
 		if ($txt =~ /^(?:diff|---|\+\+\+) /ms) {
 			$txt = undef;
-			index_diff($tg, $lines, $doc);
+			$self->index_diff($lines, $doc);
 		} else {
-			$tg->index_text($txt, 1, 'XNQ');
+			$self->index_text($txt, 1, 'XNQ');
 		}
 	} else {
-		$tg->index_text($txt, 0, 'XQUOT');
+		$self->index_text($txt, 0, 'XQUOT');
 	}
-	$tg->increase_termpos;
 	@$lines = ();
 }
 
-sub add_message {
-	# mime = Email::MIME object
-	my ($self, $mime, $bytes, $num, $oid, $mid0) = @_;
-	my $doc_id;
-	my $mids = mids($mime->header_obj);
-	my $skel = $self->{skeleton};
-
-	eval {
-		my $smsg = PublicInbox::SearchMsg->new($mime);
-		my $doc = $smsg->{doc};
-		my $subj = $smsg->subject;
-		my $xpath;
-		if ($subj ne '') {
-			$xpath = $self->subject_path($subj);
-			$xpath = id_compress($xpath);
+sub add_xapian ($$$$$) {
+	my ($self, $mime, $num, $oid, $mids, $mid0) = @_;
+	my $smsg = PublicInbox::SearchMsg->new($mime);
+	my $doc = Search::Xapian::Document->new;
+	my $subj = $smsg->subject;
+	add_val($doc, PublicInbox::Search::TS(), $smsg->ts);
+	my @ds = gmtime($smsg->ds);
+	my $yyyymmdd = strftime('%Y%m%d', @ds);
+	add_val($doc, PublicInbox::Search::YYYYMMDD(), $yyyymmdd);
+	my $dt = strftime('%Y%m%d%H%M%S', @ds);
+	add_val($doc, PublicInbox::Search::DT(), $dt);
+
+	my $tg = $self->term_generator;
+
+	$tg->set_document($doc);
+	$self->index_text($subj, 1, 'S') if $subj;
+	$self->index_users($smsg);
+
+	msg_iter($mime, sub {
+		my ($part, $depth, @idx) = @{$_[0]};
+		my $ct = $part->content_type || 'text/plain';
+		my $fn = $part->filename;
+		if (defined $fn && $fn ne '') {
+			$self->index_text($fn, 1, 'XFN');
 		}
 
-		my $lines = $mime->body_raw =~ tr!\n!\n!;
-		my @values = ($smsg->ds, $num, $bytes, $lines, $smsg->ts);
-		add_values($doc, \@values);
-
-		my $tg = $self->term_generator;
-
-		$tg->set_document($doc);
-		$tg->index_text($subj, 1, 'S') if $subj;
-		$tg->increase_termpos;
-
-		index_users($tg, $smsg);
-
-		msg_iter($mime, sub {
-			my ($part, $depth, @idx) = @{$_[0]};
-			my $ct = $part->content_type || 'text/plain';
-			my $fn = $part->filename;
-			if (defined $fn && $fn ne '') {
-				$tg->index_text($fn, 1, 'XFN');
+		my ($s, undef) = msg_part_text($part, $ct);
+		defined $s or return;
+
+		my (@orig, @quot);
+		my @lines = split(/\n/, $s);
+		while (defined(my $l = shift @lines)) {
+			if ($l =~ /^>/) {
+				$self->index_body(\@orig, $doc) if @orig;
+				push @quot, $l;
+			} else {
+				$self->index_body(\@quot, 0) if @quot;
+				push @orig, $l;
 			}
+		}
+		$self->index_body(\@quot, 0) if @quot;
+		$self->index_body(\@orig, $doc) if @orig;
+	});
 
-			return if $ct =~ m!\btext/x?html\b!i;
-
-			my $s = eval { $part->body_str };
-			if ($@) {
-				if ($ct =~ m!\btext/plain\b!i) {
-					# Try to assume UTF-8 because Alpine
-					# seems to do wacky things and set
-					# charset=X-UNKNOWN
-					$part->charset_set('UTF-8');
-					$s = eval { $part->body_str };
-					$s = $part->body if $@;
-				}
-			}
-			defined $s or return;
-
-			my (@orig, @quot);
-			my $body = $part->body;
-			my @lines = split(/\n/, $body);
-			while (defined(my $l = shift @lines)) {
-				if ($l =~ /^>/) {
-					index_body($tg, \@orig, $doc) if @orig;
-					push @quot, $l;
-				} else {
-					index_body($tg, \@quot, 0) if @quot;
-					push @orig, $l;
-				}
-			}
-			index_body($tg, \@quot, 0) if @quot;
-			index_body($tg, \@orig, $doc) if @orig;
-		});
+	foreach my $mid (@$mids) {
+		$self->index_text($mid, 1, 'XM');
 
-		# populates smsg->references for smsg->to_doc_data
-		my $refs = parse_references($smsg);
-		$mid0 = $mids->[0] unless defined $mid0; # v1 compatibility
-		my $data = $smsg->to_doc_data($oid, $mid0);
-		foreach my $mid (@$mids) {
-			$tg->index_text($mid, 1, 'XM');
+		# because too many Message-IDs are prefixed with
+		# "Pine.LNX."...
+		if ($mid =~ /\w{12,}/) {
+			my @long = ($mid =~ /(\w{3,}+)/g);
+			$self->index_text(join(' ', @long), 1, 'XM');
 		}
-		$doc->set_data($data);
-		if (my $altid = $self->{-altid}) {
-			foreach my $alt (@$altid) {
-				my $pfx = $alt->{xprefix};
-				foreach my $mid (@$mids) {
-					my $id = $alt->mid2alt($mid);
-					next unless defined $id;
-					$doc->add_boolean_term($pfx . $id);
-				}
+	}
+	$smsg->{to} = $smsg->{cc} = '';
+	PublicInbox::OverIdx::parse_references($smsg, $mid0, $mids);
+	my $data = $smsg->to_doc_data($oid, $mid0);
+	$doc->set_data($data);
+	if (my $altid = $self->{-altid}) {
+		foreach my $alt (@$altid) {
+			my $pfx = $alt->{xprefix};
+			foreach my $mid (@$mids) {
+				my $id = $alt->mid2alt($mid);
+				next unless defined $id;
+				$doc->add_boolean_term($pfx . $id);
 			}
 		}
+	}
+	$doc->add_boolean_term('Q' . $_) foreach @$mids;
+	$self->{xdb}->replace_document($num, $doc);
+}
 
-		$self->delete_article($num) if defined $num; # for reindexing
-		if ($skel) {
-			push @values, $mids, $xpath, $data;
-			$skel->index_skeleton(\@values);
-			$doc->add_boolean_term('Q' . $_) foreach @$mids;
-			$doc->add_boolean_term('XNUM' . $num) if defined $num;
-			$doc_id = $self->{xdb}->add_document($doc);
-		} else {
-			$doc_id = link_and_save($self, $doc, $mids, $refs,
-						$num, $xpath);
+sub add_message {
+	# mime = Email::MIME object
+	my ($self, $mime, $bytes, $num, $oid, $mid0) = @_;
+	my $mids = mids($mime->header_obj);
+	$mid0 = $mids->[0] unless defined $mid0; # v1 compatibility
+	unless (defined $num) { # v1
+		$self->_msgmap_init;
+		$num = index_mm($self, $mime);
+	}
+	eval {
+		if (need_xapian($self)) {
+			$self->add_xapian($mime, $num, $oid, $mids, $mid0)
+		}
+		if (my $over = $self->{over}) {
+			$over->add_overview($mime, $bytes, $num, $oid, $mid0);
 		}
 	};
 
@@ -381,7 +368,7 @@ sub add_message {
 		warn "failed to index message <".join('> <',@$mids).">: $@\n";
 		return undef;
 	}
-	$doc_id;
+	$num;
 }
 
 # returns begin and end PostingIterator
@@ -392,6 +379,7 @@ sub find_doc_ids {
 	($db->postlist_begin($termval), $db->postlist_end($termval));
 }
 
+# v1 only
 sub batch_do {
 	my ($self, $termval, $cb) = @_;
 	my $batch_size = 1000; # don't let @ids grow too large to avoid OOM
@@ -406,39 +394,43 @@ sub batch_do {
 	}
 }
 
+# v1 only, where $mid is unique
 sub remove_message {
 	my ($self, $mid) = @_;
-	my $db = $self->{xdb};
-	my $called;
 	$mid = mid_clean($mid);
 
+	if (my $over = $self->{over}) {
+		my $nr = eval { $over->remove_oid(undef, $mid) };
+		if ($@) {
+			warn "failed to remove <$mid> from overview: $@\n";
+		} elsif ($nr == 0) {
+			warn "<$mid> missing for removal from overview\n";
+		}
+	}
+	return unless need_xapian($self);
+	my $db = $self->{xdb};
+	my $nr = 0;
 	eval {
 		batch_do($self, 'Q' . $mid, sub {
 			my ($ids) = @_;
 			$db->delete_document($_) for @$ids;
-			$called = 1;
+			$nr = scalar @$ids;
 		});
 	};
 	if ($@) {
-		warn "failed to remove message <$mid>: $@\n";
-	} elsif (!$called) {
-		warn "cannot remove non-existent <$mid>\n";
+		warn "failed to remove <$mid> from Xapian: $@\n";
+	} elsif ($nr == 0) {
+		warn "<$mid> missing for removal from Xapian\n";
 	}
 }
 
-sub delete_article {
-	my ($self, $num) = @_;
-	my $ndel = 0;
-	batch_do($self, 'XNUM' . $num, sub {
-		my ($ids) = @_;
-		$ndel += scalar @$ids;
-		$self->{xdb}->delete_document($_) for @$ids;
-	});
-}
-
 # MID is a hint in V2
 sub remove_by_oid {
 	my ($self, $oid, $mid) = @_;
+
+	$self->{over}->remove_oid($oid, $mid) if $self->{over};
+
+	return unless need_xapian($self);
 	my $db = $self->{xdb};
 
 	# XXX careful, we cannot use batch_do here since we conditionally
@@ -453,9 +445,11 @@ sub remove_by_oid {
 	for (; $head != $tail; $head->inc) {
 		my $docid = $head->get_docid;
 		my $doc = $db->get_document($docid);
-		my $smsg = PublicInbox::SearchMsg->wrap($doc, $mid);
-		$smsg->load_expand;
-		push(@delete, $docid) if $smsg->{blob} eq $oid;
+		my $smsg = PublicInbox::SearchMsg->wrap($mid);
+		$smsg->load_expand($doc);
+		if ($smsg->{blob} eq $oid) {
+			push(@delete, $docid);
+		}
 	}
 	$db->delete_document($_) foreach @delete;
 	scalar(@delete);
@@ -473,105 +467,6 @@ sub term_generator { # write-only
 	$self->{term_generator} = $tg;
 }
 
-# increments last_thread_id counter
-# returns a 64-bit integer represented as a decimal string
-sub next_thread_id {
-	my ($self) = @_;
-	my $db = $self->{xdb};
-	my $last_thread_id = int($db->get_metadata('last_thread_id') || 0);
-
-	$db->set_metadata('last_thread_id', ++$last_thread_id);
-
-	$last_thread_id;
-}
-
-sub parse_references ($) {
-	my ($smsg) = @_;
-	my $mime = $smsg->{mime};
-	my $hdr = $mime->header_obj;
-	my $refs = references($hdr);
-	return $refs if scalar(@$refs) == 0;
-
-	# prevent circular references via References here:
-	my %mids = map { $_ => 1 } @{mids($hdr)};
-	my @keep;
-	foreach my $ref (@$refs) {
-		if (length($ref) > PublicInbox::MID::MAX_MID_SIZE) {
-			warn "References: <$ref> too long, ignoring\n";
-			next;
-		}
-		next if $mids{$ref};
-		push @keep, $ref;
-	}
-	$smsg->{references} = '<'.join('> <', @keep).'>' if @keep;
-	\@keep;
-}
-
-sub link_doc {
-	my ($self, $doc, $refs, $old_tid) = @_;
-	my $tid;
-
-	if (@$refs) {
-		# first ref *should* be the thread root,
-		# but we can never trust clients to do the right thing
-		my $ref = shift @$refs;
-		$tid = resolve_mid_to_tid($self, $ref);
-		merge_threads($self, $tid, $old_tid) if defined $old_tid;
-
-		# the rest of the refs should point to this tid:
-		foreach $ref (@$refs) {
-			my $ptid = resolve_mid_to_tid($self, $ref);
-			merge_threads($self, $tid, $ptid);
-		}
-	} else {
-		$tid = defined $old_tid ? $old_tid : $self->next_thread_id;
-	}
-	$doc->add_boolean_term('G' . $tid);
-	$tid;
-}
-
-sub link_and_save {
-	my ($self, $doc, $mids, $refs, $num, $xpath) = @_;
-	my $db = $self->{xdb};
-	my $old_tid;
-	my $doc_id;
-	$doc->add_boolean_term('XNUM' . $num) if defined $num;
-	$doc->add_boolean_term('XPATH' . $xpath) if defined $xpath;
-	$doc->add_boolean_term('Q' . $_) foreach @$mids;
-
-	$self->{skel} and die "Should not have read-only skel here\n";;
-	foreach my $mid (@$mids) {
-		my $vivified = 0;
-		$self->each_smsg_by_mid($mid, sub {
-			my ($cur) = @_;
-			my $type = $cur->type;
-			my $cur_tid = $cur->thread_id;
-			$old_tid = $cur_tid unless defined $old_tid;
-			if ($type eq 'mail') {
-				# do not break existing mail messages,
-				# just merge the threads
-				merge_threads($self, $old_tid, $cur_tid);
-				return 1;
-			}
-			if ($type ne 'ghost') {
-				die "<$mid> has a bad type: $type\n";
-			}
-			my $tid = link_doc($self, $doc, $refs, $old_tid);
-			$old_tid = $tid unless defined $old_tid;
-			$doc_id = $cur->{doc_id};
-			$self->{xdb}->replace_document($doc_id, $doc);
-			++$vivified;
-			1;
-		});
-		$vivified > 1 and warn
-			"BUG: vivified multiple ($vivified) ghosts for $mid\n";
-	}
-	# not really important, but we return any vivified ghost docid, here:
-	return $doc_id if defined $doc_id;
-	link_doc($self, $doc, $refs, $old_tid);
-	$self->{xdb}->add_document($doc);
-}
-
 sub index_git_blob_id {
 	my ($doc, $pfx, $objid) = @_;
 
@@ -592,8 +487,28 @@ sub index_mm {
 	my ($self, $mime) = @_;
 	my $mid = mid_clean(mid_mime($mime));
 	my $mm = $self->{mm};
-	my $num = $mm->mid_insert($mid);
-	return $num if defined $num;
+	my $num;
+
+	if (defined $self->{regen_down}) {
+		$num = $mm->num_for($mid) and return $num;
+
+		while (($num = $self->{regen_down}--) > 0) {
+			if ($mm->mid_set($num, $mid) != 0) {
+				return $num;
+			}
+		}
+	} elsif (defined $self->{regen_up}) {
+		$num = $mm->num_for($mid) and return $num;
+
+		# this is to fixup old bugs due to add-remove-add
+		while (($num = ++$self->{regen_up})) {
+			if ($mm->mid_set($num, $mid) != 0) {
+				return $num;
+			}
+		}
+	}
+
+	$num = $mm->mid_insert($mid) and return $num;
 
 	# fallback to num_for since filters like RubyLang set the number
 	$mm->num_for($mid);
@@ -604,18 +519,6 @@ sub unindex_mm {
 	$self->{mm}->mid_delete(mid_clean(mid_mime($mime)));
 }
 
-sub index_mm2 {
-	my ($self, $mime, $bytes, $blob) = @_;
-	my $num = $self->{mm}->num_for(mid_clean(mid_mime($mime)));
-	add_message($self, $mime, $bytes, $num, $blob);
-}
-
-sub unindex_mm2 {
-	my ($self, $mime) = @_;
-	$self->{mm}->mid_delete(mid_clean(mid_mime($mime)));
-	unindex_blob($self, $mime);
-}
-
 sub index_both {
 	my ($self, $mime, $bytes, $blob) = @_;
 	my $num = index_mm($self, $mime);
@@ -649,12 +552,12 @@ sub batch_adjust ($$$$) {
 	$$max -= $bytes;
 	if ($$max <= 0) {
 		$$max = BATCH_BYTES;
-		$batch_cb->($latest, 1);
+		$batch_cb->($latest);
 	}
 }
 
 # only for v1
-sub rlog {
+sub read_log {
 	my ($self, $log, $add_cb, $del_cb, $batch_cb) = @_;
 	my $hex = '[a-f0-9]';
 	my $h40 = $hex .'{40}';
@@ -665,178 +568,189 @@ sub rlog {
 	my $bytes;
 	my $max = BATCH_BYTES;
 	local $/ = "\n";
+	my %D;
 	my $line;
+	my $newest;
 	while (defined($line = <$log>)) {
 		if ($line =~ /$addmsg/o) {
 			my $blob = $1;
+			if (delete $D{$blob}) {
+				if (defined $self->{regen_down}) {
+					my $num = $self->{regen_down}--;
+					$self->{mm}->num_highwater($num);
+				}
+				next;
+			}
 			my $mime = do_cat_mail($git, $blob, \$bytes) or next;
 			batch_adjust(\$max, $bytes, $batch_cb, $latest);
 			$add_cb->($self, $mime, $bytes, $blob);
 		} elsif ($line =~ /$delmsg/o) {
 			my $blob = $1;
-			my $mime = do_cat_mail($git, $blob, \$bytes) or next;
-			batch_adjust(\$max, $bytes, $batch_cb, $latest);
-			$del_cb->($self, $mime);
+			$D{$blob} = 1;
 		} elsif ($line =~ /^commit ($h40)/o) {
 			$latest = $1;
+			$newest ||= $latest;
 		}
 	}
-	$batch_cb->($latest, 0);
+	# get the leftovers
+	foreach my $blob (keys %D) {
+		my $mime = do_cat_mail($git, $blob, \$bytes) or next;
+		$del_cb->($self, $mime);
+	}
+	$batch_cb->($latest, $newest);
 }
 
 sub _msgmap_init {
 	my ($self) = @_;
+	die "BUG: _msgmap_init is only for v1\n" if $self->{version} != 1;
 	$self->{mm} ||= eval {
 		require PublicInbox::Msgmap;
-		my $msgmap_path = $self->{msgmap_path};
-		if (defined $msgmap_path) { # v2
-			PublicInbox::Msgmap->new_file($msgmap_path, 1);
-		} else {
-			PublicInbox::Msgmap->new($self->{mainrepo}, 1);
-		}
+		PublicInbox::Msgmap->new($self->{mainrepo}, 1);
 	};
 }
 
 sub _git_log {
 	my ($self, $range) = @_;
-	$self->{git}->popen(qw/log --reverse --no-notes --no-color
+	my $git = $self->{git};
+
+	if (index($range, '..') < 0) {
+		# don't show annoying git errrors to users who run -index
+		# on empty inboxes
+		$git->qx(qw(rev-parse -q --verify), "$range^0");
+		if ($?) {
+			open my $fh, '<', '/dev/null' or
+				die "failed to open /dev/null: $!\n";
+			return $fh;
+		}
+	}
+
+	# Count the new files so they can be added newest to oldest
+	# and still have numbers increasing from oldest to newest
+	my $fcount = 0;
+	# can't use 'rev-list --count' if we use --diff-filter
+	my $fh = $git->popen(qw(log --pretty=tformat:%h
+			     --no-notes --no-color --no-renames
+			     --diff-filter=AM), $range);
+	++$fcount while <$fh>;
+	my $high = $self->{mm}->num_highwater;
+
+	if (index($range, '..') < 0) {
+		if ($high && $high == $fcount) {
+			# fix up old bugs in full indexes which caused messages to
+			# not appear in Msgmap
+			$self->{regen_up} = $high;
+		} else {
+			# normal regen is for for fresh data
+			$self->{regen_down} = $fcount;
+		}
+	} else {
+		# Give oldest messages the smallest numbers
+		$self->{regen_down} = $high + $fcount;
+	}
+
+	$git->popen(qw/log --no-notes --no-color --no-renames
 				--raw -r --no-abbrev/, $range);
 }
 
-# indexes all unindexed messages
+# --is-ancestor requires git 1.8.0+
+sub is_ancestor ($$$) {
+	my ($git, $cur, $tip) = @_;
+	return 0 unless $git->check($cur);
+	my $cmd = [ 'git', "--git-dir=$git->{git_dir}",
+		qw(merge-base --is-ancestor), $cur, $tip ];
+	my $pid = spawn($cmd);
+	defined $pid or die "spawning ".join(' ', @$cmd)." failed: $!";
+	waitpid($pid, 0) == $pid or die join(' ', @$cmd) .' did not finish';
+	$? == 0;
+}
+
+sub need_update ($$$) {
+	my ($self, $cur, $new) = @_;
+	my $git = $self->{git};
+	return 1 if $cur && !is_ancestor($git, $cur, $new);
+	my $range = $cur eq '' ? $new : "$cur..$new";
+	chomp(my $n = $git->qx(qw(rev-list --count), $range));
+	($n eq '' || $n > 0);
+}
+
+# The last git commit we indexed with Xapian or SQLite (msgmap)
+# This needs to account for cases where Xapian or SQLite is
+# out-of-date with respect to the other.
+sub _last_x_commit {
+	my ($self, $mm) = @_;
+	my $lm = $mm->last_commit || '';
+	my $lx = '';
+	if (need_xapian($self)) {
+		$lx = $self->{xdb}->get_metadata('last_commit') || '';
+	} else {
+		$lx = $lm;
+	}
+	# Use last_commit from msgmap if it is older or unset
+	if (!$lm || ($lx && $lx && is_ancestor($self->{git}, $lm, $lx))) {
+		$lx = $lm;
+	}
+	$lx;
+}
+
+# indexes all unindexed messages (v1 only)
 sub _index_sync {
 	my ($self, $opts) = @_;
 	my $tip = $opts->{ref} || 'HEAD';
-	my $reindex = $opts->{reindex};
-	my ($mkey, $last_commit, $lx, $xlog);
-	$self->{git}->batch_prepare;
-	my $xdb = _xdb_acquire($self);
-	$xdb->begin_transaction;
+	my ($last_commit, $lx, $xlog);
+	my $git = $self->{git};
+	$git->batch_prepare;
+
+	my $xdb = $self->begin_txn_lazy;
+	my $mm = _msgmap_init($self);
 	do {
 		$xlog = undef;
-		$mkey = 'last_commit';
-		$last_commit = $xdb->get_metadata('last_commit');
-		$lx = $last_commit;
-		if ($reindex) {
-			$lx = '';
-			$mkey = undef if $last_commit ne '';
-		}
-		$xdb->cancel_transaction;
+		$last_commit = _last_x_commit($self, $mm);
+		$lx = $opts->{reindex} ? '' : $last_commit;
+
+		$self->{over}->rollback_lazy;
+		$self->{over}->disconnect;
+		$git->cleanup;
+		delete $self->{txn};
+		$xdb->cancel_transaction if $xdb;
 		$xdb = _xdb_release($self);
 
-		# ensure we leak no FDs to "git log"
+		# ensure we leak no FDs to "git log" with Xapian <= 1.2
 		my $range = $lx eq '' ? $tip : "$lx..$tip";
 		$xlog = _git_log($self, $range);
 
-		$xdb = _xdb_acquire($self);
-		$xdb->begin_transaction;
-	} while ($xdb->get_metadata('last_commit') ne $last_commit);
+		$xdb = $self->begin_txn_lazy;
+	} while (_last_x_commit($self, $mm) ne $last_commit);
 
-	my $mm = _msgmap_init($self);
 	my $dbh = $mm->{dbh} if $mm;
-	my $mm_only;
 	my $cb = sub {
-		my ($commit, $more) = @_;
+		my ($commit, $newest) = @_;
 		if ($dbh) {
-			$mm->last_commit($commit) if $commit;
+			if ($newest) {
+				my $cur = $mm->last_commit || '';
+				if (need_update($self, $cur, $newest)) {
+					$mm->last_commit($newest);
+				}
+			}
 			$dbh->commit;
 		}
-		if (!$mm_only) {
-			$xdb->set_metadata($mkey, $commit) if $mkey && $commit;
-			$xdb->commit_transaction;
-			$xdb = _xdb_release($self);
+		if ($newest && need_xapian($self)) {
+			my $cur = $xdb->get_metadata('last_commit');
+			if (need_update($self, $cur, $newest)) {
+				$xdb->set_metadata('last_commit', $newest);
+			}
 		}
+		$self->commit_txn_lazy;
+		$git->cleanup;
+		$xdb = _xdb_release($self);
 		# let another process do some work... <
-		if ($more) {
-			if (!$mm_only) {
-				$xdb = _xdb_acquire($self);
-				$xdb->begin_transaction;
-			}
+		if (!$newest) {
+			$xdb = $self->begin_txn_lazy;
 			$dbh->begin_work if $dbh;
 		}
 	};
 
-	if ($mm) {
-		$dbh->begin_work;
-		my $lm = $mm->last_commit || '';
-		if ($lm eq $lx) {
-			# Common case is the indexes are synced,
-			# we only need to run git-log once:
-			rlog($self, $xlog, *index_both, *unindex_both, $cb);
-		} else {
-			# Uncommon case, msgmap and xapian are out-of-sync
-			# do not care for performance (but git is fast :>)
-			# This happens if we have to reindex Xapian since
-			# msgmap is a frozen format and our Xapian format
-			# is evolving.
-			my $r = $lm eq '' ? $tip : "$lm..$tip";
-
-			# first, ensure msgmap is up-to-date:
-			my $mkey_prev = $mkey;
-			$mkey = undef; # ignore xapian, for now
-			my $mlog = _git_log($self, $r);
-			$mm_only = 1;
-			rlog($self, $mlog, *index_mm, *unindex_mm, $cb);
-			$mm_only = $mlog = undef;
-
-			# now deal with Xapian
-			$mkey = $mkey_prev;
-			$dbh = undef;
-			rlog($self, $xlog, *index_mm2, *unindex_mm2, $cb);
-		}
-	} else {
-		# user didn't install DBD::SQLite and DBI
-		rlog($self, $xlog, *add_message, *unindex_blob, $cb);
-	}
-}
-
-# this will create a ghost as necessary
-sub resolve_mid_to_tid {
-	my ($self, $mid) = @_;
-	my $tid;
-	$self->each_smsg_by_mid($mid, sub {
-		my ($smsg) = @_;
-		my $cur_tid = $smsg->thread_id;
-		if (defined $tid) {
-			merge_threads($self, $tid, $cur_tid);
-		} else {
-			$tid = $smsg->thread_id;
-		}
-		1;
-	});
-	return $tid if defined $tid;
-
-	$self->create_ghost($mid)->thread_id;
-}
-
-sub create_ghost {
-	my ($self, $mid) = @_;
-
-	my $tid = $self->next_thread_id;
-	my $doc = Search::Xapian::Document->new;
-	$doc->add_boolean_term('Q' . $mid);
-	$doc->add_boolean_term('G' . $tid);
-	$doc->add_boolean_term('T' . 'ghost');
-
-	my $smsg = PublicInbox::SearchMsg->wrap($doc, $mid);
-	$self->{xdb}->add_document($doc);
-
-	$smsg;
-}
-
-sub merge_threads {
-	my ($self, $winner_tid, $loser_tid) = @_;
-	return if $winner_tid == $loser_tid;
-	my $db = $self->{xdb};
-	batch_do($self, 'G' . $loser_tid, sub {
-		my ($ids) = @_;
-		foreach my $docid (@$ids) {
-			my $doc = $db->get_document($docid);
-			$doc->remove_term('G' . $loser_tid);
-			$doc->add_boolean_term('G' . $winner_tid);
-			$db->replace_document($docid, $doc);
-		}
-	});
+	$dbh->begin_work;
+	read_log($self, $xlog, *index_both, *unindex_both, $cb);
 }
 
 sub DESTROY {
@@ -845,16 +759,13 @@ sub DESTROY {
 	$_[0]->{lockfh} = undef;
 }
 
-# remote_* subs are only used by SearchIdxPart and SearchIdxSkeleton
+# remote_* subs are only used by SearchIdxPart
 sub remote_commit {
 	my ($self) = @_;
 	if (my $w = $self->{w}) {
 		print $w "commit\n" or die "failed to write commit: $!";
 	} else {
 		$self->commit_txn_lazy;
-		if (my $skel = $self->{skeleton}) {
-			$skel->commit_txn_lazy;
-		}
 	}
 }
 
@@ -875,7 +786,7 @@ sub remote_close {
 sub remote_remove {
 	my ($self, $oid, $mid) = @_;
 	if (my $w = $self->{w}) {
-		# triggers remove_by_oid in partition or skeleton
+		# triggers remove_by_oid in a partition
 		print $w "D $oid $mid\n" or die "failed to write remove $!";
 	} else {
 		$self->begin_txn_lazy;
@@ -886,20 +797,32 @@ sub remote_remove {
 sub begin_txn_lazy {
 	my ($self) = @_;
 	return if $self->{txn};
-	my $xdb = $self->{xdb} || $self->_xdb_acquire;
-	$xdb->begin_transaction;
-	$self->{txn} = 1;
+
+	$self->{-inbox}->with_umask(sub {
+		my $xdb = $self->{xdb} || $self->_xdb_acquire;
+		$self->{over}->begin_lazy if $self->{over};
+		$xdb->begin_transaction if $xdb;
+		$self->{txn} = 1;
+		$xdb;
+	});
 }
 
 sub commit_txn_lazy {
 	my ($self) = @_;
 	delete $self->{txn} or return;
-	$self->{xdb}->commit_transaction;
+	$self->{-inbox}->with_umask(sub {
+		if (my $xdb = $self->{xdb}) {
+			$xdb->commit_transaction;
+		}
+		$self->{over}->commit_lazy if $self->{over};
+	});
 }
 
 sub worker_done {
 	my ($self) = @_;
-	die "$$ $0 xdb not released\n" if $self->{xdb};
+	if (need_xapian($self)) {
+		die "$$ $0 xdb not released\n" if $self->{xdb};
+	}
 	die "$$ $0 still in transaction\n" if $self->{txn};
 }