X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FAdmin.pm;h=fb88e621b021b8b68e47c1524fd9227634abf81a;hb=256b3566e4d57a64bb349a7b896da10381a2a363;hp=af2b3da95c3406bfe3dd53fdd062b08895dd6f3a;hpb=e3944dfcfb3c0d8964d7cd458a9f14bbf503d00f;p=public-inbox.git

diff --git a/lib/PublicInbox/Admin.pm b/lib/PublicInbox/Admin.pm
index af2b3da9..fb88e621 100644
--- a/lib/PublicInbox/Admin.pm
+++ b/lib/PublicInbox/Admin.pm
@@ -48,13 +48,14 @@ sub resolve_repo_dir {
 sub detect_indexlevel ($) {
 	my ($ibx) = @_;
 
-	# brand new or never before indexed inboxes default to full
-	return 'full' unless $ibx->over;
-	delete $ibx->{over}; # don't leave open FD lying around
+	my $over = $ibx->over;
+	my $srch = $ibx->search;
+	delete @$ibx{qw(over search)}; # don't leave open FDs lying around
 
+	# brand new or never before indexed inboxes default to full
+	return 'full' unless $over;
 	my $l = 'basic';
-	my $srch = $ibx->search or return $l;
-	delete $ibx->{search}; # don't leave open FD lying around
+	return $l unless $srch;
 	if (my $xdb = $srch->xdb) {
 		$l = 'full';
 		my $m = $xdb->get_metadata('indexlevel');
@@ -65,6 +66,7 @@ sub detect_indexlevel ($) {
 $ibx->{inboxdir} has unexpected indexlevel in Xapian: $m
 
 		}
+		$ibx->{-skip_docdata} = 1 if $xdb->get_metadata('skip_docdata');
 	}
 	$l;
 }
@@ -129,7 +131,7 @@ EOF
 		}
 	}
 	if (@old) {
-		die "inboxes $min_ver inboxes not supported by $0\n\t",
+		die "-V$min_ver inboxes not supported by $0\n\t",
 		    join("\n\t", @old), "\n";
 	}
 	@ibxs;
@@ -207,6 +209,9 @@ sub index_terminate {
 sub index_inbox {
 	my ($ibx, $im, $opt) = @_;
 	my $jobs = delete $opt->{jobs} if $opt;
+	if (my $pr = $opt->{-progress}) {
+		$pr->("indexing $ibx->{inboxdir} ...\n");
+	}
 	local %SIG = %SIG;
 	setup_signals(\&index_terminate, $ibx);
 	if (ref($ibx) && $ibx->version == 2) {
@@ -219,9 +224,9 @@ sub index_inbox {
 				$v2w->{parallel} = 0;
 			} else {
 				my $n = $v2w->{shards};
-				if ($jobs != ($n + 1) && !$opt->{reshard}) {
+				if ($jobs < ($n + 1) && !$opt->{reshard}) {
 					warn
-"Unable to respect --jobs=$jobs, inbox was created with $n shards\n";
+"Unable to respect --jobs=$jobs on index, inbox was created with $n shards\n";
 				}
 			}
 		}
@@ -256,13 +261,49 @@ sub progress_prepare ($) {
 
 # same unit factors as git:
 sub parse_unsigned ($) {
-	my ($max_size) = @_;
+	my ($val) = @_;
 
-	$$max_size =~ /\A([0-9]+)([kmg])?\z/i or return;
+	$$val =~ /\A([0-9]+)([kmg])?\z/i or return;
 	my ($n, $unit_factor) = ($1, $2 // '');
 	my %u = ( k => 1024, m => 1024**2, g => 1024**3 );
-	$$max_size = $n * ($u{lc($unit_factor)} // 1);
+	$$val = $n * ($u{lc($unit_factor)} // 1);
 	1;
 }
 
+sub index_prepare ($$) {
+	my ($opt, $cfg) = @_;
+	my $env;
+	if ($opt->{compact}) {
+		require PublicInbox::Xapcmd;
+		PublicInbox::Xapcmd::check_compact();
+		$opt->{compact_opt} = { -coarse_lock => 1, compact => 1 };
+		if (defined(my $jobs = $opt->{jobs})) {
+			$opt->{compact_opt}->{jobs} = $jobs;
+		}
+	}
+	for my $k (qw(max_size batch_size)) {
+		my $git_key = "publicInbox.index".ucfirst($k);
+		$git_key =~ s/_([a-z])/\U$1/g;
+		defined(my $v = $opt->{$k} // $cfg->{lc($git_key)}) or next;
+		parse_unsigned(\$v) or die "`$git_key=$v' not parsed\n";
+		$v > 0 or die "`$git_key=$v' must be positive\n";
+		$opt->{$k} = $v;
+	}
+
+	# out-of-the-box builds of Xapian 1.4.x are still limited to 32-bit
+	# https://getting-started-with-xapian.readthedocs.io/en/latest/concepts/indexing/limitations.html
+	$opt->{batch_size} and
+		$env = { XAPIAN_FLUSH_THRESHOLD => '4294967295' };
+
+	for my $k (qw(sequential_shard)) {
+		my $git_key = "publicInbox.index".ucfirst($k);
+		$git_key =~ s/_([a-z])/\U$1/g;
+		defined(my $s = $opt->{$k} // $cfg->{lc($git_key)}) or next;
+		defined(my $v = $cfg->git_bool($s))
+					or die "`$git_key=$s' not boolean\n";
+		$opt->{$k} = $v;
+	}
+	$env;
+}
+
 1;