admin: progress shows the inbox being indexed

[public-inbox.git] / lib / PublicInbox / Admin.pm

diff --git a/lib/PublicInbox/Admin.pm b/lib/PublicInbox/Admin.pm
index 8a9a81c91180bd0f315a5440df7ba01df1adfe4c..f5427af77376a6e1da40f5fe9aa187c19a53f36e 100644 (file)
--- a/lib/PublicInbox/Admin.pm
+++ b/lib/PublicInbox/Admin.pm
@@ -207,6 +207,9 @@ sub index_terminate {
 sub index_inbox {
        my ($ibx, $im, $opt) = @_;
        my $jobs = delete $opt->{jobs} if $opt;
+       if (my $pr = $opt->{-progress}) {
+               $pr->("indexing $ibx->{inboxdir} ...\n");
+       }
        local %SIG = %SIG;
        setup_signals(\&index_terminate, $ibx);
        if (ref($ibx) && $ibx->version == 2) {
@@ -219,9 +222,9 @@ sub index_inbox {
                                $v2w->{parallel} = 0;
                        } else {
                                my $n = $v2w->{shards};
-                               if ($jobs != ($n + 1) && !$opt->{reshard}) {
+                               if ($jobs < ($n + 1) && !$opt->{reshard}) {
                                        warn
-"Unable to respect --jobs=$jobs, inbox was created with $n shards\n";
+"Unable to respect --jobs=$jobs on index, inbox was created with $n shards\n";
                                }
                        }
                }
@@ -265,4 +268,40 @@ sub parse_unsigned ($) {
        1;
 }
 
+sub index_prepare ($$) {
+       my ($opt, $cfg) = @_;
+       my $env;
+       if ($opt->{compact}) {
+               require PublicInbox::Xapcmd;
+               PublicInbox::Xapcmd::check_compact();
+               $opt->{compact_opt} = { -coarse_lock => 1, compact => 1 };
+               if (defined(my $jobs = $opt->{jobs})) {
+                       $opt->{compact_opt}->{jobs} = $jobs;
+               }
+       }
+       for my $k (qw(max_size batch_size)) {
+               my $git_key = "publicInbox.index".ucfirst($k);
+               $git_key =~ s/_([a-z])/\U$1/g;
+               defined(my $v = $opt->{$k} // $cfg->{lc($git_key)}) or next;
+               parse_unsigned(\$v) or die "`$git_key=$v' not parsed\n";
+               $v > 0 or die "`$git_key=$v' must be positive\n";
+               $opt->{$k} = $v;
+       }
+
+       # out-of-the-box builds of Xapian 1.4.x are still limited to 32-bit
+       # https://getting-started-with-xapian.readthedocs.io/en/latest/concepts/indexing/limitations.html
+       $opt->{batch_size} and
+               $env = { XAPIAN_FLUSH_THRESHOLD => '4294967295' };
+
+       for my $k (qw(sequential_shard)) {
+               my $git_key = "publicInbox.index".ucfirst($k);
+               $git_key =~ s/_([a-z])/\U$1/g;
+               defined(my $s = $opt->{$k} // $cfg->{lc($git_key)}) or next;
+               defined(my $v = $cfg->git_bool($s))
+                                       or die "`$git_key=$s' not boolean\n";
+               $opt->{$k} = $v;
+       }
+       $env;
+}
+
 1;