X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FSearchIdx.pm;h=b2d71a1f88b462493d3db527c609d00f4e16b31b;hb=e67dc3e9fc73029332a632d022002d3ce80c1306;hp=0aeeb6bc544acce630adacd28e6b063e8295b6be;hpb=0c6d38221ec9fbf9d7c7e0329252b10f17ab7a27;p=public-inbox.git

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 0aeeb6bc..b2d71a1f 100644
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2018 all contributors <meta@public-inbox.org>
+# Copyright (C) 2015-2019 all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 # based on notmuch, but with no concept of folders, files or flags
 #
@@ -12,7 +12,7 @@ use warnings;
 use base qw(PublicInbox::Search PublicInbox::Lock);
 use PublicInbox::MIME;
 use PublicInbox::InboxWritable;
-use PublicInbox::MID qw/mid_clean id_compress mid_mime mids/;
+use PublicInbox::MID qw/mid_clean id_compress mid_mime mids_for_index/;
 use PublicInbox::MsgIter;
 use Carp qw(croak);
 use POSIX qw(strftime);
@@ -29,36 +29,27 @@ use constant {
 my $xapianlevels = qr/\A(?:full|medium)\z/;
 
 sub new {
-	my ($class, $ibx, $creat, $part) = @_;
+	my ($class, $ibx, $creat, $shard) = @_;
+	ref $ibx or die "BUG: expected PublicInbox::Inbox object: $ibx";
 	my $levels = qr/\A(?:full|medium|basic)\z/;
-	my $mainrepo = $ibx; # for "public-inbox-index" w/o entry in config
-	my $git_dir = $mainrepo;
-	my ($altid, $git);
-	my $version = 1;
+	my $inboxdir = $ibx->{inboxdir};
+	my $version = $ibx->{version} || 1;
 	my $indexlevel = 'full';
-	if (ref $ibx) {
-		$mainrepo = $ibx->{mainrepo};
-		$altid = $ibx->{altid};
-		$version = $ibx->{version} || 1;
-		if ($altid) {
-			require PublicInbox::AltId;
-			$altid = [ map {
-				PublicInbox::AltId->new($ibx, $_);
-			} @$altid ];
-		}
-		if ($ibx->{indexlevel}) {
-			if ($ibx->{indexlevel} =~ $levels) {
-				$indexlevel = $ibx->{indexlevel};
-			} else {
-				die("Invalid indexlevel $ibx->{indexlevel}\n");
-			}
+	my $altid = $ibx->{altid};
+	if ($altid) {
+		require PublicInbox::AltId;
+		$altid = [ map { PublicInbox::AltId->new($ibx, $_); } @$altid ];
+	}
+	if ($ibx->{indexlevel}) {
+		if ($ibx->{indexlevel} =~ $levels) {
+			$indexlevel = $ibx->{indexlevel};
+		} else {
+			die("Invalid indexlevel $ibx->{indexlevel}\n");
 		}
-	} else { # FIXME: old tests: old tests
-		$ibx = { mainrepo => $git_dir, version => 1 };
 	}
 	$ibx = PublicInbox::InboxWritable->new($ibx);
 	my $self = bless {
-		mainrepo => $mainrepo,
+		inboxdir => $inboxdir,
 		-inbox => $ibx,
 		git => $ibx->git,
 		-altid => $altid,
@@ -67,13 +58,13 @@ sub new {
 	}, $class;
 	$ibx->umask_prepare;
 	if ($version == 1) {
-		$self->{lock_path} = "$mainrepo/ssoma.lock";
+		$self->{lock_path} = "$inboxdir/ssoma.lock";
 		my $dir = $self->xdir;
 		$self->{over} = PublicInbox::OverIdx->new("$dir/over.sqlite3");
 	} elsif ($version == 2) {
-		defined $part or die "partition is required for v2\n";
-		# partition is a number
-		$self->{partition} = $part;
+		defined $shard or die "shard is required for v2\n";
+		# shard is a number
+		$self->{shard} = $shard;
 		$self->{lock_path} = undef;
 	} else {
 		die "unsupported inbox version=$version\n";
@@ -111,13 +102,17 @@ sub _xdb_acquire {
 		$self->lock_acquire;
 
 		# don't create empty Xapian directories if we don't need Xapian
-		my $is_part = defined($self->{partition});
-		if (!$is_part || ($is_part && need_xapian($self))) {
+		my $is_shard = defined($self->{shard});
+		if (!$is_shard || ($is_shard && need_xapian($self))) {
 			File::Path::mkpath($dir);
 		}
 	}
 	return unless defined $flag;
-	$self->{xdb} = Search::Xapian::WritableDatabase->new($dir, $flag);
+	my $xdb = eval { Search::Xapian::WritableDatabase->new($dir, $flag) };
+	if ($@) {
+		die "Failed opening $dir: ", $@;
+	}
+	$self->{xdb} = $xdb;
 }
 
 sub add_val ($$$) {
@@ -349,7 +344,7 @@ sub add_xapian ($$$$$) {
 sub add_message {
 	# mime = Email::MIME object
 	my ($self, $mime, $bytes, $num, $oid, $mid0) = @_;
-	my $mids = mids($mime->header_obj);
+	my $mids = mids_for_index($mime->header_obj);
 	$mid0 = $mids->[0] unless defined $mid0; # v1 compatibility
 	unless (defined $num) { # v1
 		$self->_msgmap_init;
@@ -549,12 +544,12 @@ sub index_sync {
 	$self->{-inbox}->with_umask(sub { $self->_index_sync($opts) })
 }
 
-sub batch_adjust ($$$$) {
-	my ($max, $bytes, $batch_cb, $latest) = @_;
+sub batch_adjust ($$$$$) {
+	my ($max, $bytes, $batch_cb, $latest, $nr) = @_;
 	$$max -= $bytes;
 	if ($$max <= 0) {
 		$$max = BATCH_BYTES;
-		$batch_cb->($latest);
+		$batch_cb->($nr, $latest);
 	}
 }
 
@@ -573,6 +568,7 @@ sub read_log {
 	my %D;
 	my $line;
 	my $newest;
+	my $nr = 0;
 	while (defined($line = <$log>)) {
 		if ($line =~ /$addmsg/o) {
 			my $blob = $1;
@@ -584,7 +580,7 @@ sub read_log {
 				next;
 			}
 			my $mime = do_cat_mail($git, $blob, \$bytes) or next;
-			batch_adjust(\$max, $bytes, $batch_cb, $latest);
+			batch_adjust(\$max, $bytes, $batch_cb, $latest, ++$nr);
 			$add_cb->($self, $mime, $bytes, $blob);
 		} elsif ($line =~ /$delmsg/o) {
 			my $blob = $1;
@@ -599,7 +595,7 @@ sub read_log {
 		my $mime = do_cat_mail($git, $blob, \$bytes) or next;
 		$del_cb->($self, $mime);
 	}
-	$batch_cb->($latest, $newest);
+	$batch_cb->($nr, $latest, $newest);
 }
 
 sub _msgmap_init {
@@ -607,12 +603,12 @@ sub _msgmap_init {
 	die "BUG: _msgmap_init is only for v1\n" if $self->{version} != 1;
 	$self->{mm} ||= eval {
 		require PublicInbox::Msgmap;
-		PublicInbox::Msgmap->new($self->{mainrepo}, 1);
+		PublicInbox::Msgmap->new($self->{inboxdir}, 1);
 	};
 }
 
 sub _git_log {
-	my ($self, $range) = @_;
+	my ($self, $opts, $range) = @_;
 	my $git = $self->{git};
 
 	if (index($range, '..') < 0) {
@@ -629,12 +625,17 @@ sub _git_log {
 	# Count the new files so they can be added newest to oldest
 	# and still have numbers increasing from oldest to newest
 	my $fcount = 0;
+	my $pr = $opts->{-progress};
+	$pr->("counting changes\n\t$range ... ") if $pr;
 	# can't use 'rev-list --count' if we use --diff-filter
 	my $fh = $git->popen(qw(log --pretty=tformat:%h
 			     --no-notes --no-color --no-renames
 			     --diff-filter=AM), $range);
 	++$fcount while <$fh>;
+	close $fh;
 	my $high = $self->{mm}->num_highwater;
+	$pr->("$fcount\n") if $pr; # continue previous line
+	$self->{ntodo} = $fcount;
 
 	if (index($range, '..') < 0) {
 		if ($high && $high == $fcount) {
@@ -688,7 +689,7 @@ sub _last_x_commit {
 		$lx = $lm;
 	}
 	# Use last_commit from msgmap if it is older or unset
-	if (!$lm || ($lx && $lx && is_ancestor($self->{git}, $lm, $lx))) {
+	if (!$lm || ($lx && $lm && is_ancestor($self->{git}, $lm, $lx))) {
 		$lx = $lm;
 	}
 	$lx;
@@ -707,6 +708,7 @@ sub _index_sync {
 	my ($last_commit, $lx, $xlog);
 	my $git = $self->{git};
 	$git->batch_prepare;
+	my $pr = $opts->{-progress};
 
 	my $xdb = $self->begin_txn_lazy;
 	my $mm = _msgmap_init($self);
@@ -724,14 +726,14 @@ sub _index_sync {
 
 		# ensure we leak no FDs to "git log" with Xapian <= 1.2
 		my $range = $lx eq '' ? $tip : "$lx..$tip";
-		$xlog = _git_log($self, $range);
+		$xlog = _git_log($self, $opts, $range);
 
 		$xdb = $self->begin_txn_lazy;
 	} while (_last_x_commit($self, $mm) ne $last_commit);
 
 	my $dbh = $mm->{dbh} if $mm;
 	my $cb = sub {
-		my ($commit, $newest) = @_;
+		my ($nr, $commit, $newest) = @_;
 		if ($dbh) {
 			if ($newest) {
 				my $cur = $mm->last_commit || '';
@@ -751,6 +753,7 @@ sub _index_sync {
 		$git->cleanup;
 		$xdb = _xdb_release($self);
 		# let another process do some work... <
+		$pr->("indexed $nr/$self->{ntodo}\n") if $pr && $nr;
 		if (!$newest) {
 			$xdb = $self->begin_txn_lazy;
 			$dbh->begin_work if $dbh;
@@ -794,7 +797,7 @@ sub remote_close {
 sub remote_remove {
 	my ($self, $oid, $mid) = @_;
 	if (my $w = $self->{w}) {
-		# triggers remove_by_oid in a partition
+		# triggers remove_by_oid in a shard
 		print $w "D $oid $mid\n" or die "failed to write remove $!";
 	} else {
 		$self->begin_txn_lazy;
@@ -820,6 +823,15 @@ sub commit_txn_lazy {
 	delete $self->{txn} or return;
 	$self->{-inbox}->with_umask(sub {
 		if (my $xdb = $self->{xdb}) {
+
+			# store 'indexlevel=medium' in v2 shard=0 and
+			# v1 (only one shard)
+			# This metadata is read by Admin::detect_indexlevel:
+			if (!$self->{shard} # undef or 0, not >0
+			    && $self->{indexlevel} eq 'medium') {
+				$xdb->set_metadata('indexlevel', 'medium');
+			}
+
 			$xdb->commit_transaction;
 		}
 		$self->{over}->commit_lazy if $self->{over};