SearchIdx: Allow the amount of indexing be configured

This adds a new inbox configuration option 'indexlevel' that can take
the values 'full', 'medium', and 'basic'.

When set to 'full' everything is indexed including the positions
of all terms.

When set to 'medium' everything except the positions of terms is
indexed.

When set to 'basic' terms and positions are not indexed.  Just the
Overview database for NNTP is created.  Which is still quite good and
allows searching for messages by Message-ID.  But there are no indexes to support
searching inside the email messages themselves.

Update the reindex tests to exercise the full medium and basic code paths

Signed-off-by: "Eric W. Biederman" <ebiederm@xmission.com>

diff --git a/lib/PublicInbox/Config.pm b/lib/PublicInbox/Config.pm
index 289c36a6ccf1a6c6e5518cd4dff12f4e8cd63fce..78586560cf0f7a20e38c3752b536a20e03c6301d 100644 (file)
--- a/lib/PublicInbox/Config.pm
+++ b/lib/PublicInbox/Config.pm
@@ -158,7 +158,7 @@ sub _fill {
 
        foreach my $k (qw(mainrepo filter url newsgroup
                        infourl watch watchheader httpbackendmax
-                       replyto feedmax nntpserver)) {
+                       replyto feedmax nntpserver indexlevel)) {
                my $v = $self->{"$pfx.$k"};
                $rv->{$k} = $v if defined $v;
        }

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 8978914ab087b3d6e25c69b988e490f9ea71fd91..04e853069f0661c24b5d223d2c1ad72f47e7232b 100644 (file)
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -47,6 +47,7 @@ sub git_unquote ($) {
 
 sub new {
        my ($class, $ibx, $creat, $part) = @_;
+       my $levels = qr/\A(?:full|medium|basic)\z/;
        my $mainrepo = $ibx; # for "public-inbox-index" w/o entry in config
        my $git_dir = $mainrepo;
        my ($altid, $git);
@@ -62,6 +63,13 @@ sub new {
                                PublicInbox::AltId->new($ibx, $_);
                        } @$altid ];
                }
+               if ($ibx->{indexlevel}) {
+                       if ($ibx->{indexlevel} =~ $levels) {
+                               $indexlevel = $ibx->{indexlevel};
+                       } else {
+                               die("Invalid indexlevel $ibx->{indexlevel}\n");
+                       }
+               }
        } else { # v1
                $ibx = { mainrepo => $git_dir, version => 1 };
        }

diff --git a/t/v1reindex.t b/t/v1reindex.t
index 0df36d3fee893f6d2d38eee6c56e013c004b3ad8..ff32750fc3c3a1b07aff08aa34c420be40066af6 100644 (file)
--- a/t/v1reindex.t
+++ b/t/v1reindex.t
@@ -20,6 +20,7 @@ my $ibx_config = {
        mainrepo => $mainrepo,
        name => 'test-v1reindex',
        -primary_address => 'test@example.com',
+       indexlevel => 'full',
 };
 my $ibx = PublicInbox::Inbox->new($ibx_config);
 my $mime = PublicInbox::MIME->create(
@@ -74,15 +75,32 @@ is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
 
 ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
 remove_tree($xap);
+ok(!-d $xap, 'Xapian directories removed again');
+
 $rw = PublicInbox::SearchIdx->new($ibx, 1);
+{
+       my @warn;
+       local $SIG{__WARN__} = sub { push @warn, @_ };
+       eval { $rw->index_sync({reindex => 1}) };
+       is($@, '', 'no error from reindexing without msgmap');
+       is(scalar(@warn), 0, 'no warnings from reindexing');
+       $im->done;
+       ok(-d $xap, 'Xapian directories recreated');
+       delete $ibx->{mm};
+       is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
+}
 
+ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
+remove_tree($xap);
 ok(!-d $xap, 'Xapian directories removed again');
+
+$rw = PublicInbox::SearchIdx->new($ibx, 1);
 {
        my @warn;
        local $SIG{__WARN__} = sub { push @warn, @_ };
        eval { $rw->index_sync({reindex => 1}) };
        is($@, '', 'no error from reindexing without msgmap');
-       is(scalar(@warn), 0, 'no warnings from reindexing');
+       is_deeply(\@warn, [], 'no warnings');
        $im->done;
        ok(-d $xap, 'Xapian directories recreated');
        delete $ibx->{mm};
@@ -91,9 +109,31 @@ ok(!-d $xap, 'Xapian directories removed again');
 
 ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
 remove_tree($xap);
+ok(!-d $xap, 'Xapian directories removed again');
+
+$ibx_config->{index_level} = 'medium';
+$ibx = PublicInbox::Inbox->new($ibx_config);
 $rw = PublicInbox::SearchIdx->new($ibx, 1);
+{
+       my @warn;
+       local $SIG{__WARN__} = sub { push @warn, @_ };
+       eval { $rw->index_sync({reindex => 1}) };
+       is($@, '', 'no error from reindexing without msgmap');
+       is_deeply(\@warn, [], 'no warnings');
+       $im->done;
+       ok(-d $xap, 'Xapian directories recreated');
+       delete $ibx->{mm};
+       is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
+}
 
+
+ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
+remove_tree($xap);
 ok(!-d $xap, 'Xapian directories removed again');
+
+$ibx_config->{index_level} = 'basic';
+$ibx = PublicInbox::Inbox->new($ibx_config);
+$rw = PublicInbox::SearchIdx->new($ibx, 1);
 {
        my @warn;
        local $SIG{__WARN__} = sub { push @warn, @_ };
@@ -106,4 +146,5 @@ ok(!-d $xap, 'Xapian directories removed again');
        is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
 }
 
+
 done_testing();

diff --git a/t/v2reindex.t b/t/v2reindex.t
index 8af30991f858d52537f9813b64a98945f7162227..20903967bf255f7133c09d71821165417003f3e6 100644 (file)
--- a/t/v2reindex.t
+++ b/t/v2reindex.t
@@ -19,6 +19,7 @@ my $ibx_config = {
        name => 'test-v2writable',
        version => 2,
        -primary_address => 'test@example.com',
+       indexlevel => 'full',
 };
 my $ibx = PublicInbox::Inbox->new($ibx_config);
 my $mime = PublicInbox::MIME->create(
@@ -95,4 +96,43 @@ ok(!-d $xap, 'Xapian directories removed again');
        is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
 }
 
+ok(unlink "$mainrepo/msgmap.sqlite3", 'remove msgmap');
+remove_tree($xap);
+ok(!-d $xap, 'Xapian directories removed again');
+
+$ibx_config->{index_level} = 'medium';
+$ibx = PublicInbox::Inbox->new($ibx_config);
+$im = PublicInbox::V2Writable->new($ibx);
+{
+       my @warn;
+       local $SIG{__WARN__} = sub { push @warn, @_ };
+       eval { $im->index_sync({reindex => 1}) };
+       is($@, '', 'no error from reindexing without msgmap');
+       is_deeply(\@warn, [], 'no warnings');
+       $im->done;
+       ok(-d $xap, 'Xapian directories recreated');
+       delete $ibx->{mm};
+       is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
+}
+
+
+ok(unlink "$mainrepo/msgmap.sqlite3", 'remove msgmap');
+remove_tree($xap);
+ok(!-d $xap, 'Xapian directories removed again');
+
+$ibx_config->{index_level} = 'basic';
+$ibx = PublicInbox::Inbox->new($ibx_config);
+$im = PublicInbox::V2Writable->new($ibx);
+{
+       my @warn;
+       local $SIG{__WARN__} = sub { push @warn, @_ };
+       eval { $im->index_sync({reindex => 1}) };
+       is($@, '', 'no error from reindexing without msgmap');
+       is_deeply(\@warn, [], 'no warnings');
+       $im->done;
+       ok(-d $xap, 'Xapian directories recreated');
+       delete $ibx->{mm};
+       is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
+}
+
 done_testing();