searchidx: store indexlevel=medium as metadata

And use it from Admin.

It's easy to tell what indexlevel=basic is from unconfigured
inboxes, but distinguishing between 'medium' and 'full' would
require stat()-ing position.* files which is fragile and
Xapian-implementation-dependent.

So use the metadata facility of Xapian and store it in the main
partition so Admin tools can deal better with unconfigured
inboxes copied using generic tools like cp(1) or rsync(1).

diff --git a/lib/PublicInbox/Admin.pm b/lib/PublicInbox/Admin.pm
index 07d8b572c3d3075e9f7ef94620ca19d3096bc2a7..4a862c6d3c5af28c4d01e246516e286449605f5b 100644 (file)
--- a/lib/PublicInbox/Admin.pm
+++ b/lib/PublicInbox/Admin.pm
@@ -41,6 +41,31 @@ sub resolve_repo_dir {
        }
 }
 
+# for unconfigured inboxes
+sub detect_indexlevel ($) {
+       my ($ibx) = @_;
+
+       # brand new or never before indexed inboxes default to full
+       return 'full' unless $ibx->over;
+       delete $ibx->{over}; # don't leave open FD lying around
+
+       my $l = 'basic';
+       my $srch = $ibx->search or return $l;
+       delete $ibx->{search}; # don't leave open FD lying around
+       if (my $xdb = $srch->xdb) {
+               $l = 'full';
+               my $m = $xdb->get_metadata('indexlevel');
+               if ($m eq 'medium') {
+                       $l = $m;
+               } elsif ($m ne '') {
+                       warn <<"";
+$ibx->{mainrepo} has unexpected indexlevel in Xapian: $m
+
+               }
+       }
+       $l;
+}
+
 sub resolve_inboxes {
        my ($argv, $warn_on_unconfigured) = @_;
        require PublicInbox::Config;

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index b963805ee237314d0aefc2a346b89bbebc683039..99856286adb13d0085e16a636df95d80f501e49a 100644 (file)
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -828,6 +828,14 @@ sub commit_txn_lazy {
        delete $self->{txn} or return;
        $self->{-inbox}->with_umask(sub {
                if (my $xdb = $self->{xdb}) {
+
+                       # store 'indexlevel=medium' in v2 part=0 and v1 (only part)
+                       # This metadata is read by Admin::detect_indexlevel:
+                       if (!$self->{partition} # undef or 0, not >0
+                           && $self->{indexlevel} eq 'medium') {
+                               $xdb->set_metadata('indexlevel', 'medium');
+                       }
+
                        $xdb->commit_transaction;
                }
                $self->{over}->commit_lazy if $self->{over};

diff --git a/lib/PublicInbox/Xapcmd.pm b/lib/PublicInbox/Xapcmd.pm
index 7e3d47fb9b26240e4fc563afcbf01ccb61b61b60..906723104d553d2cb25465c4dfd488e213914a0f 100644 (file)
--- a/lib/PublicInbox/Xapcmd.pm
+++ b/lib/PublicInbox/Xapcmd.pm
@@ -275,6 +275,14 @@ sub cpdb ($$) {
                        my $lc = $src->get_metadata('last_commit');
                        $dst->set_metadata('last_commit', $lc) if $lc;
 
+                       # only the first xapian partition (0) gets 'indexlevel'
+                       if ($old =~ m!(?:xapian\d+|xap\d+/0)\z!) {
+                               my $l = $src->get_metadata('indexlevel');
+                               if ($l eq 'medium') {
+                                       $dst->set_metadata('indexlevel', $l);
+                               }
+                       }
+
                        $it = $src->postlist_begin('');
                        $end = $src->postlist_end('');
                        if ($pr) {

diff --git a/script/public-inbox-index b/script/public-inbox-index
index e4a7be1979ae88915d0ef689ed99c5ccdf51d347..439da15708645c792c0d485818b5611e89a2bc11 100755 (executable)
--- a/script/public-inbox-index
+++ b/script/public-inbox-index
@@ -25,10 +25,9 @@ PublicInbox::Admin::require_or_die('-index');
 usage() unless @ibxs;
 my $mods = {};
 foreach my $ibx (@ibxs) {
-       if (defined $opt->{indexlevel} && !defined($ibx->{indexlevel})) {
-               # XXX: users can shoot themselves in the foot, with this...
-               $ibx->{indexlevel} = $opt->{indexlevel};
-       }
+       # XXX: users can shoot themselves in the foot, with opt->{indexlevel}
+       $ibx->{indexlevel} //= $opt->{indexlevel} //
+                       PublicInbox::Admin::detect_indexlevel($ibx);
        PublicInbox::Admin::scan_ibx_modules($mods, $ibx);
 }
 

diff --git a/t/indexlevels-mirror.t b/t/indexlevels-mirror.t
index 125113689a51121df5b739aa8f7813582bc7bfc2..bf0f801822adf4970dae3e1bccdc0f9794b8e254 100644 (file)
--- a/t/indexlevels-mirror.t
+++ b/t/indexlevels-mirror.t
@@ -7,6 +7,7 @@ use PublicInbox::MIME;
 use PublicInbox::Inbox;
 use PublicInbox::InboxWritable;
 use File::Temp qw/tempdir/;
+require PublicInbox::Admin;
 require './t/common.perl';
 require_git(2.6);
 my $this = (split('/', __FILE__))[-1];
@@ -119,6 +120,8 @@ sub import_index_incremental {
 
        if ($level ne 'basic') {
                is(system(@xcpdb, $mirror), 0, "v$v xcpdb OK");
+               is(PublicInbox::Admin::detect_indexlevel($ro_mirror), $level,
+                  'indexlevel detectable by Admin after xcpdb v' .$v.$level);
                delete $ro_mirror->{$_} for (qw(over search));
                ($nr, $msgs) = $ro_mirror->search->query('m:m@2');
                is($nr, 1, "v$v found m\@2 via Xapian on $level");
@@ -157,6 +160,9 @@ sub import_index_incremental {
        @rw_nums = map { $_->{num} } @{$ibx->over->query_ts(0, 0)};
        is_deeply(\@rw_nums, \@expect, "v$v master has expected NNTP articles");
        is_deeply(\@ro_nums, \@expect, "v$v mirror matches master articles");
+
+       is(PublicInbox::Admin::detect_indexlevel($ro_mirror), $level,
+          'indexlevel detectable by Admin '.$v.$level);
 }
 
 # we can probably cull some other tests and put full/medium tests, here
@@ -172,9 +178,11 @@ SKIP: {
        require PublicInbox::Search;
        PublicInbox::Search::load_xapian() or skip 'Search::Xapian missing', 2;
        for my $v (1..2) {
-               subtest("v$v indexlevel=medium" => sub {
-                       import_index_incremental($v, 'medium');
-               })
+               foreach my $l (qw(medium full)) {
+                       subtest("v$v indexlevel=$l" => sub {
+                               import_index_incremental($v, $l);
+                       });
+               }
        }
 }