lazy load Xapian and make it optional for v2

More tests work without Search::Xapian, now.
Usability issues still need to be fixed

diff --git a/INSTALL b/INSTALL
index 313a295151883ed580ea5dac7186b95c40053402..72e0763c417166d78381087a6b5d0dab336a4d91 100644 (file)
--- a/INSTALL
+++ b/INSTALL
@@ -69,7 +69,7 @@ Numerous optional modules are likely to be useful as well:
 - Search::Xapian                   deb: libsearch-xapian-perl
                                    pkg: p5-Search-Xapian
                                    rpm: perl-Search-Xapian
-                                   (for v2, HTTP search)
+                                   (HTTP search)
 
 - Net::Server                      deb: libnet-server-perl
                                    pkg: pkg-Net-Server

diff --git a/TODO b/TODO
index d947b0ff822198abccc4f1c558eedd42f64bbae5..4953439e8c85bdd101a7b394888023c6706c2955 100644 (file)
--- a/TODO
+++ b/TODO
@@ -69,8 +69,6 @@ all need to be considered for everything we introduce)
 
 * large mbox/Maildir/MH/NNTP spool import (see PublicInbox::Import)
 
-* Allow NNTP and more of PSGI code to work without Xapian
-
 * Read-only WebDAV interface to the git repo so it can be mounted
   via davfs2 or fusedav to avoid full clones.
   davfs2 needs Range: request support for this to be feasible:

diff --git a/ci/deps.perl b/ci/deps.perl
index e0fda01191c0be44d61ab876febdb01377607afe..32b0226e82a4e5efaa453404844d248b28abc91a 100755 (executable)
--- a/ci/deps.perl
+++ b/ci/deps.perl
@@ -70,12 +70,7 @@ if (@precious) {
 
 
 # bare minimum for v2
-$profiles->{v2essential} = [ @{$profiles->{essential}}, qw(
-       DBD::SQLite
-       DBI
-       Search::Xapian
-       xapian-compact
-) ];
+$profiles->{v2essential} = [ @{$profiles->{essential}}, qw(DBD::SQLite DBI) ];
 
 # package names which can't be mapped automatically:
 my $non_auto = {

diff --git a/ci/profiles.sh b/ci/profiles.sh
index 31140f35827dc380160614ce7d4934ae12b8dffc..5d7c2cf6d2297dddf80e5dbc1dc361ce9ed1cda0 100755 (executable)
--- a/ci/profiles.sh
+++ b/ci/profiles.sh
@@ -62,8 +62,9 @@ essential devtest-
 EOF
        ;;
 debian-sid|debian-9|debian-10) sed "s/^/$PKG_FMT /" <<EOF
-all devtest-
 all devtest
+all devtest Search::Xapian-
+all devtest-
 v2essential
 essential
 essential devtest-

diff --git a/lib/PublicInbox/Inbox.pm b/lib/PublicInbox/Inbox.pm
index dc186b731da8a41bd231ba9edc9594b1642566e0..813ed997f1781ccc3791257f751124fdf5ee9021 100644 (file)
--- a/lib/PublicInbox/Inbox.pm
+++ b/lib/PublicInbox/Inbox.pm
@@ -25,7 +25,7 @@ sub cleanup_task () {
        my $next = {};
        for my $ibx (values %$CLEANUP) {
                my $again;
-               foreach my $f (qw(mm search)) {
+               foreach my $f (qw(mm search over)) {
                        delete $ibx->{$f} if SvREFCNT($ibx->{$f}) == 1;
                }
                my $expire = time - 60;
@@ -37,7 +37,7 @@ sub cleanup_task () {
                                $again = 1 if $git->cleanup($expire);
                        }
                }
-               $again ||= !!($ibx->{mm} || $ibx->{search});
+               $again ||= !!($ibx->{over} || $ibx->{mm} || $ibx->{search});
                $next->{"$ibx"} = $ibx if $again;
        }
        $CLEANUP = $next;
@@ -175,14 +175,17 @@ sub search ($;$) {
                require PublicInbox::Search;
                PublicInbox::Search->new($self, $self->{altid});
        };
-       # TODO: lazily load Xapian
-       # return $srch if $over_only || eval { $srch->xdb };
-       # undef;
+       ($over_only || eval { $srch->xdb }) ? $srch : undef;
 }
 
 sub over ($) {
-       my $srch = search($_[0], 1) or return;
-       $srch->{over_ro};
+       my ($self) = @_;
+       my $srch = search($self, 1) or return;
+       $self->{over} ||= eval {
+               my $over = $srch->{over_ro};
+               $over->dbh_new; # may fail
+               $over;
+       }
 }
 
 sub try_cat {
@@ -290,7 +293,7 @@ sub nntp_url {
 sub nntp_usable {
        my ($self) = @_;
        my $ret = mm($self) && over($self);
-       $self->{mm} = $self->{search} = undef;
+       $self->{mm} = $self->{over} = $self->{search} = undef;
        $ret;
 }
 

diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm
index e79ec0f80a0ead18b122108220bd6b1b80c13757..b1e62f4cb0d3589327a87fb4396f6efb52ba9e2e 100644 (file)
--- a/lib/PublicInbox/Search.pm
+++ b/lib/PublicInbox/Search.pm
@@ -12,11 +12,21 @@ use constant TS => 0;  # Received: header in Unix time
 use constant YYYYMMDD => 1; # Date: header for searching in the WWW UI
 use constant DT => 2; # Date: YYYYMMDDHHMMSS
 
-use Search::Xapian qw/:standard/;
 use PublicInbox::SearchMsg;
 use PublicInbox::MIME;
 use PublicInbox::MID qw/id_compress/;
 use PublicInbox::Over;
+my $QP_FLAGS;
+sub load_xapian () {
+       $QP_FLAGS ||= eval {
+               require Search::Xapian;
+               Search::Xapian->import(qw(:standard));
+
+               # n.b. FLAG_PURE_NOT is expensive not suitable for a public
+               # website as it could become a denial-of-service vector
+               FLAG_PHRASE()|FLAG_BOOLEAN()|FLAG_LOVEHATE()|FLAG_WILDCARD();
+       };
+};
 
 # This is English-only, everything else is non-standard and may be confused as
 # a prefix common in patch emails
@@ -41,10 +51,6 @@ use constant {
        #      (commit 83425ef12e4b65cdcecd11ddcb38175d4a91d5a0)
        # 14 - fix ghost root vivification
        SCHEMA_VERSION => 15,
-
-       # n.b. FLAG_PURE_NOT is expensive not suitable for a public website
-       # as it could become a denial-of-service vector
-       QP_FLAGS => FLAG_PHRASE|FLAG_BOOLEAN|FLAG_LOVEHATE|FLAG_WILDCARD,
 };
 
 my %bool_pfx_external = (
@@ -113,18 +119,43 @@ EOF
 );
 chomp @HELP;
 
-sub xdir {
-       my ($self) = @_;
+sub xdir ($;$) {
+       my ($self, $rdonly) = @_;
        if ($self->{version} == 1) {
                "$self->{mainrepo}/public-inbox/xapian" . SCHEMA_VERSION;
        } else {
                my $dir = "$self->{mainrepo}/xap" . SCHEMA_VERSION;
+               return $dir if $rdonly;
+
                my $part = $self->{partition};
                defined $part or die "partition not given";
                $dir .= "/$part";
        }
 }
 
+sub xdb ($) {
+       my ($self) = @_;
+       $self->{xdb} ||= do {
+               load_xapian();
+               my $dir = xdir($self, 1);
+               if ($self->{version} >= 2) {
+                       my $xdb;
+                       foreach my $part (<$dir/*>) {
+                               -d $part && $part =~ m!/\d+\z! or next;
+                               my $sub = Search::Xapian::Database->new($part);
+                               if ($xdb) {
+                                       $xdb->add_database($sub);
+                               } else {
+                                       $xdb = $sub;
+                               }
+                       }
+                       $xdb;
+               } else {
+                       Search::Xapian::Database->new($dir);
+               }
+       };
+}
+
 sub new {
        my ($class, $mainrepo, $altid) = @_;
        my $version = 1;
@@ -138,33 +169,16 @@ sub new {
                altid => $altid,
                version => $version,
        }, $class;
-       my $dir;
-       if ($version >= 2) {
-               $dir = "$self->{mainrepo}/xap" . SCHEMA_VERSION;
-               my $xdb;
-               my $parts = 0;
-               foreach my $part (<$dir/*>) {
-                       -d $part && $part =~ m!/\d+\z! or next;
-                       $parts++;
-                       my $sub = Search::Xapian::Database->new($part);
-                       if ($xdb) {
-                               $xdb->add_database($sub);
-                       } else {
-                               $xdb = $sub;
-                       }
-               }
-               $self->{xdb} = $xdb;
-       } else {
-               $dir = $self->xdir;
-               $self->{xdb} = Search::Xapian::Database->new($dir);
-       }
+       my $dir = xdir($self, 1);
        $self->{over_ro} = PublicInbox::Over->new("$dir/over.sqlite3");
        $self;
 }
 
 sub reopen {
        my ($self) = @_;
-       $self->{xdb}->reopen;
+       if (my $xdb = $self->{xdb}) {
+               $xdb->reopen;
+       }
        $self; # make chaining easier
 }
 
@@ -175,7 +189,8 @@ sub query {
        if ($query_string eq '' && !$opts->{mset}) {
                $self->{over_ro}->recent($opts);
        } else {
-               my $query = $self->qp->parse_query($query_string, QP_FLAGS);
+               my $qp = qp($self);
+               my $query = $qp->parse_query($query_string, $QP_FLAGS);
                $opts->{relevance} = 1 unless exists $opts->{relevance};
                _do_enquire($self, $query, $opts);
        }
@@ -213,7 +228,8 @@ sub _do_enquire {
 
 sub _enquire_once {
        my ($self, $query, $opts) = @_;
-       my $enquire = Search::Xapian::Enquire->new($self->{xdb});
+       my $xdb = xdb($self);
+       my $enquire = Search::Xapian::Enquire->new($xdb);
        $enquire->set_query($query);
        $opts ||= {};
         my $desc = !$opts->{asc};
@@ -246,13 +262,13 @@ sub qp {
 
        my $qp = $self->{query_parser};
        return $qp if $qp;
-
+       my $xdb = xdb($self);
        # new parser
        $qp = Search::Xapian::QueryParser->new;
-       $qp->set_default_op(OP_AND);
-       $qp->set_database($self->{xdb});
+       $qp->set_default_op(OP_AND());
+       $qp->set_database($xdb);
        $qp->set_stemmer($self->stemmer);
-       $qp->set_stemming_strategy(STEM_SOME);
+       $qp->set_stemming_strategy(STEM_SOME());
        $qp->set_max_wildcard_expansion(100);
        $qp->add_valuerangeprocessor(
                Search::Xapian::NumberValueRangeProcessor->new(YYYYMMDD, 'd:'));

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 1b86f7278e785ecd88dbb5d83eafd33adb0d656b..135b5eb9b0f1768abbe0b0c115e6fbaf2e9cef39 100644 (file)
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -54,11 +54,10 @@ sub new {
                                die("Invalid indexlevel $ibx->{indexlevel}\n");
                        }
                }
-       } else { # v1
+       } else { # FIXME: old tests: old tests
                $ibx = { mainrepo => $git_dir, version => 1 };
        }
        $ibx = PublicInbox::InboxWritable->new($ibx);
-       require Search::Xapian::WritableDatabase;
        my $self = bless {
                mainrepo => $mainrepo,
                -inbox => $ibx,
@@ -84,25 +83,36 @@ sub new {
        $self;
 }
 
+sub need_xapian ($) { $_[0]->{indexlevel} =~ $xapianlevels }
+
 sub _xdb_release {
        my ($self) = @_;
-       my $xdb = delete $self->{xdb} or croak 'not acquired';
-       $xdb->close;
+       if (need_xapian($self)) {
+               my $xdb = delete $self->{xdb} or croak 'not acquired';
+               $xdb->close;
+       }
        $self->lock_release if $self->{creat};
        undef;
 }
 
 sub _xdb_acquire {
        my ($self) = @_;
-       croak 'already acquired' if $self->{xdb};
+       my $flag;
        my $dir = $self->xdir;
-       my $flag = Search::Xapian::DB_OPEN;
+       if (need_xapian($self)) {
+               croak 'already acquired' if $self->{xdb};
+               PublicInbox::Search::load_xapian();
+               require Search::Xapian::WritableDatabase;
+               $flag = $self->{creat} ?
+                       Search::Xapian::DB_CREATE_OR_OPEN() :
+                       Search::Xapian::DB_OPEN();
+       }
        if ($self->{creat}) {
                require File::Path;
                $self->lock_acquire;
                File::Path::mkpath($dir);
-               $flag = Search::Xapian::DB_CREATE_OR_OPEN;
        }
+       return unless defined $flag;
        $self->{xdb} = Search::Xapian::WritableDatabase->new($dir, $flag);
 }
 
@@ -342,7 +352,7 @@ sub add_message {
                $num = index_mm($self, $mime);
        }
        eval {
-               if ($self->{indexlevel} =~ $xapianlevels) {
+               if (need_xapian($self)) {
                        $self->add_xapian($mime, $num, $oid, $mids, $mid0)
                }
                if (my $over = $self->{over}) {
@@ -383,7 +393,6 @@ sub batch_do {
 # v1 only, where $mid is unique
 sub remove_message {
        my ($self, $mid) = @_;
-       my $db = $self->{xdb};
        $mid = mid_clean($mid);
 
        if (my $over = $self->{over}) {
@@ -394,7 +403,8 @@ sub remove_message {
                        warn "<$mid> missing for removal from overview\n";
                }
        }
-       return if $self->{indexlevel} !~ $xapianlevels;
+       return unless need_xapian($self);
+       my $db = $self->{xdb};
        my $nr = 0;
        eval {
                batch_do($self, 'Q' . $mid, sub {
@@ -413,10 +423,12 @@ sub remove_message {
 # MID is a hint in V2
 sub remove_by_oid {
        my ($self, $oid, $mid) = @_;
-       my $db = $self->{xdb};
 
        $self->{over}->remove_oid($oid, $mid) if $self->{over};
 
+       return unless need_xapian($self);
+       my $db = $self->{xdb};
+
        # XXX careful, we cannot use batch_do here since we conditionally
        # delete documents based on other factors, so we cannot call
        # find_doc_ids twice.
@@ -664,7 +676,7 @@ sub _last_x_commit {
        my ($self, $mm) = @_;
        my $lm = $mm->last_commit || '';
        my $lx = '';
-       if ($self->{indexlevel} =~ $xapianlevels) {
+       if (need_xapian($self)) {
                $lx = $self->{xdb}->get_metadata('last_commit') || '';
        } else {
                $lx = $lm;
@@ -695,7 +707,7 @@ sub _index_sync {
                $self->{over}->disconnect;
                $git->cleanup;
                delete $self->{txn};
-               $xdb->cancel_transaction;
+               $xdb->cancel_transaction if $xdb;
                $xdb = _xdb_release($self);
 
                # ensure we leak no FDs to "git log" with Xapian <= 1.2
@@ -717,7 +729,7 @@ sub _index_sync {
                        }
                        $dbh->commit;
                }
-               if ($newest && $self->{indexlevel} =~ $xapianlevels) {
+               if ($newest && need_xapian($self)) {
                        my $cur = $xdb->get_metadata('last_commit');
                        if (need_update($self, $cur, $newest)) {
                                $xdb->set_metadata('last_commit', $newest);
@@ -785,7 +797,7 @@ sub begin_txn_lazy {
        $self->{-inbox}->with_umask(sub {
                my $xdb = $self->{xdb} || $self->_xdb_acquire;
                $self->{over}->begin_lazy if $self->{over};
-               $xdb->begin_transaction;
+               $xdb->begin_transaction if $xdb;
                $self->{txn} = 1;
                $xdb;
        });
@@ -795,14 +807,18 @@ sub commit_txn_lazy {
        my ($self) = @_;
        delete $self->{txn} or return;
        $self->{-inbox}->with_umask(sub {
-               $self->{xdb}->commit_transaction;
+               if (my $xdb = $self->{xdb}) {
+                       $xdb->commit_transaction;
+               }
                $self->{over}->commit_lazy if $self->{over};
        });
 }
 
 sub worker_done {
        my ($self) = @_;
-       die "$$ $0 xdb not released\n" if $self->{xdb};
+       if (need_xapian($self)) {
+               die "$$ $0 xdb not released\n" if $self->{xdb};
+       }
        die "$$ $0 still in transaction\n" if $self->{txn};
 }
 

diff --git a/lib/PublicInbox/WWW.pm b/lib/PublicInbox/WWW.pm
index 0f963dcbd907af3097ed5d0a8462c20acee3a0d9..8e1b1afe9c6c6872be0fa8fc9f195a81ad75a3a8 100644 (file)
--- a/lib/PublicInbox/WWW.pm
+++ b/lib/PublicInbox/WWW.pm
@@ -149,8 +149,11 @@ sub preload {
        require PublicInbox::MIME;
        require Digest::SHA;
        require POSIX;
-
-       foreach (qw(PublicInbox::Search PublicInbox::SearchView
+       eval {
+               require PublicInbox::Search;
+               PublicInbox::Search::load_xapian();
+       };
+       foreach (qw(PublicInbox::SearchView
                        PublicInbox::Mbox IO::Compress::Gzip
                        PublicInbox::NewsWWW)) {
                eval "require $_;";

diff --git a/script/public-inbox-index b/script/public-inbox-index
index b353093e49250e2296f6c12edaee656dd40fb064..53def9a7253d918f9a68db705baea8fd9fdfcffd 100755 (executable)
--- a/script/public-inbox-index
+++ b/script/public-inbox-index
@@ -26,10 +26,12 @@ if ($@) {
 my $reindex;
 my $prune;
 my $jobs = undef;
+my $indexlevel;
 my %opts = (
        '--reindex' => \$reindex,
        '--jobs|j=i' => \$jobs,
        '--prune' => \$prune,
+        'L|indexlevel=s' => \$indexlevel,
 );
 GetOptions(%opts) or die "bad command-line args\n$usage";
 die "--jobs must be positive\n" if defined $jobs && $jobs < 0;
@@ -55,18 +57,27 @@ defined($config) and $config->each_inbox(sub {
 });
 
 foreach my $dir (@dirs) {
-       if (!ref($dir) && -f "$dir/inbox.lock") { # v2
-               my $ibx = { mainrepo => $dir, name => 'unnamed' };
+       if (!ref($dir)) {
+               unless (-d $dir) {
+                       die "$dir does not appear to be an inbox repository\n";
+               }
+               my $ibx = {
+                       mainrepo => $dir,
+                       name => 'unnamed',
+                       indexlevel => $indexlevel,
+                       version => -f "$dir/inbox.lock" ? 2 : 1,
+               };
                $dir = PublicInbox::Inbox->new($ibx);
+       } elsif (defined $indexlevel && !defined($dir->{indexlevel})) {
+               # XXX: users can shoot themselves in the foot, with this...
+               $dir->{indexlevel} = $indexlevel;
        }
-       index_dir($dir);
+
+       index_inbox($dir);
 }
 
-sub index_dir {
+sub index_inbox {
        my ($repo) = @_;
-       if (!ref $repo && ! -d $repo) {
-               die "$repo does not appear to be an inbox repository\n";
-       }
        if (ref($repo) && ($repo->{version} || 1) == 2) {
                eval { require PublicInbox::V2Writable };
                die "v2 requirements not met: $@\n" if $@;

diff --git a/script/public-inbox-init b/script/public-inbox-init
index 8bb7845114feadb3164dc1796e489a7b096febae..9f0bd1bddca8b1d773d6563a916c87b7f1697ec0 100755 (executable)
--- a/script/public-inbox-init
+++ b/script/public-inbox-init
@@ -109,6 +109,7 @@ my $ibx = PublicInbox::Inbox->new({
        name => $name,
        version => $version,
        -primary_address => $address[0],
+       indexlevel => $indexlevel,
 });
 
 if ($version >= 2) {

diff --git a/script/public-inbox-purge b/script/public-inbox-purge
index 688dd9501b6d35cb517145adca2c7247b67fb76a..264bcdefa2c88bb06b16457281c614acf51a1bef 100755 (executable)
--- a/script/public-inbox-purge
+++ b/script/public-inbox-purge
@@ -63,6 +63,11 @@ if ($all) {
                                mainrepo => $dir,
                        });
                };
+
+               # somebody could "rm -r" all the Xapian directories;
+               # let them purge the overview, at least
+               $ibx->{indexlevel} ||= 'basic' unless $ibx->search;
+
                push @inboxes, $ibx;
        }
 

diff --git a/t/admin.t b/t/admin.t
index b5a4383ec67d84b08c6e11c4e012bb5a65e7288e..3790c9e2e20a2248e22c259e63b027bc346244e6 100644 (file)
--- a/t/admin.t
+++ b/t/admin.t
@@ -50,7 +50,7 @@ SKIP: {
 
 # v2
 SKIP: {
-       for my $m (qw(DBD::SQLite Search::Xapian)) {
+       for my $m (qw(DBD::SQLite)) {
                skip "$m missing", 5 unless eval "require $m";
        }
        use_ok 'PublicInbox::V2Writable';

diff --git a/t/cgi.t b/t/cgi.t
index b24bbc4ac5f244f0e66bea26f32c463d35ae2087..d3172bf74cbd1820e2ae2104ac95c712fd3ae328 100644 (file)
--- a/t/cgi.t
+++ b/t/cgi.t
@@ -30,6 +30,7 @@ my $cfgpfx = "publicinbox.test";
        my %cfg = (
                "$cfgpfx.address" => $addr,
                "$cfgpfx.mainrepo" => $maindir,
+               "$cfgpfx.indexlevel" => 'basic',
        );
        while (my ($k,$v) = each %cfg) {
                is(0, system(qw(git config --file), $pi_config, $k, $v),
@@ -39,9 +40,12 @@ my $cfgpfx = "publicinbox.test";
 
 use_ok 'PublicInbox::Git';
 use_ok 'PublicInbox::Import';
-use_ok 'Email::MIME';
-my $git = PublicInbox::Git->new($maindir);
-my $im = PublicInbox::Import->new($git, 'test', $addr);
+use_ok 'PublicInbox::Inbox';
+use_ok 'PublicInbox::V1Writable';
+use_ok 'PublicInbox::Config';
+my $cfg = PublicInbox::Config->new($pi_config);
+my $ibx = $cfg->lookup_name('test');
+my $im = PublicInbox::V1Writable->new($ibx);
 
 {
        local $ENV{HOME} = $home;
@@ -103,8 +107,9 @@ EOF
        like($res->{head}, qr/^Status: 501 /, "search not-yet-enabled");
        my $indexed;
        eval {
+               require DBD::SQLite;
                require PublicInbox::SearchIdx;
-               my $s = PublicInbox::SearchIdx->new($maindir, 1);
+               my $s = PublicInbox::SearchIdx->new($ibx, 1);
                $s->index_sync;
                $indexed = 1;
        };
@@ -120,6 +125,7 @@ EOF
                };
        } else {
                like($res->{head}, qr/^Status: 501 /, "search not available");
+               SKIP: { skip 'DBD::SQLite not available', 2 };
        }
 
        my $have_xml_feed = eval { require XML::Feed; 1 } if $indexed;
@@ -132,6 +138,8 @@ EOF
                my $p = XML::Feed->parse(\($res->{body}));
                is($p->format, "Atom", "parsed atom feed");
                is(scalar $p->entries, 3, "parsed three entries");
+       } else {
+               SKIP: { skip 'DBD::SQLite or XML::Feed missing', 2 };
        }
 }
 

diff --git a/t/indexlevels-mirror.t b/t/indexlevels-mirror.t
index e25b827f7cfa3f71012d82e99eaa962441440abe..cac7050bdca3840c484cc778c3d5484e892652f3 100644 (file)
--- a/t/indexlevels-mirror.t
+++ b/t/indexlevels-mirror.t
@@ -10,8 +10,7 @@ require './t/common.perl';
 require_git(2.6);
 my $this = (split('/', __FILE__))[-1];
 
-# TODO: remove Search::Xapian as a requirement for basic
-foreach my $mod (qw(DBD::SQLite Search::Xapian)) {
+foreach my $mod (qw(DBD::SQLite)) {
        eval "require $mod";
        plan skip_all => "$mod missing for $this" if $@;
 }
@@ -47,8 +46,11 @@ sub import_index_incremental {
        $im->done;
 
        # index master (required for v1)
-       is(system($index, $ibx->{mainrepo}), 0, 'index master OK');
-       my $ro_master = PublicInbox::Inbox->new({mainrepo => $ibx->{mainrepo}});
+       is(system($index, $ibx->{mainrepo}, "-L$level"), 0, 'index master OK');
+       my $ro_master = PublicInbox::Inbox->new({
+               mainrepo => $ibx->{mainrepo},
+               indexlevel => $level
+       });
        my ($nr, $msgs) = $ro_master->recent;
        is($nr, 1, 'only one message in master, so far');
        is($msgs->[0]->{mid}, 'm@1', 'first message in master indexed');
@@ -75,7 +77,10 @@ sub import_index_incremental {
        is(system($index, $mirror), 0, "v$v index mirror OK");
 
        # read-only access
-       my $ro_mirror = PublicInbox::Inbox->new({mainrepo => $mirror});
+       my $ro_mirror = PublicInbox::Inbox->new({
+               mainrepo => $mirror,
+               indexlevel => 'basic'
+       });
        ($nr, $msgs) = $ro_mirror->recent;
        is($nr, 1, 'only one message, so far');
        is($msgs->[0]->{mid}, 'm@1', 'read first message');
@@ -94,7 +99,7 @@ sub import_index_incremental {
                ['m@1','m@2'], 'got both messages in mirror');
 
        # incremental index master (required for v1)
-       is(system($index, $ibx->{mainrepo}), 0, 'index master OK');
+       is(system($index, $ibx->{mainrepo}, "-L$level"), 0, 'index master OK');
        ($nr, $msgs) = $ro_master->recent;
        is($nr, 2, '2nd message seen in master');
        is_deeply([sort { $a cmp $b } map { $_->{mid} } @$msgs],

diff --git a/t/nntp.t b/t/nntp.t
index c4b9737c7a556a9f582b0609ed4f915ac33a4c28..7ab801c1c2bdcb4f8ba2bd6985c56f91d6ca387b 100644 (file)
--- a/t/nntp.t
+++ b/t/nntp.t
@@ -4,7 +4,7 @@ use strict;
 use warnings;
 use Test::More;
 
-foreach my $mod (qw(DBD::SQLite Search::Xapian Data::Dumper)) {
+foreach my $mod (qw(DBD::SQLite Data::Dumper)) {
        eval "require $mod";
        plan skip_all => "$mod missing for nntp.t" if $@;
 }

diff --git a/t/nntpd.t b/t/nntpd.t
index b8a84a0c2c003b00ae681e8cd0bda94e8c57f517..c7ea3197c20f3808fc037e5144ee5aad2fd89c5f 100644 (file)
--- a/t/nntpd.t
+++ b/t/nntpd.t
@@ -3,7 +3,7 @@
 use strict;
 use warnings;
 use Test::More;
-foreach my $mod (qw(DBD::SQLite Search::Xapian)) {
+foreach my $mod (qw(DBD::SQLite)) {
        eval "require $mod";
        plan skip_all => "$mod missing for nntpd.t" if $@;
 }
@@ -55,12 +55,13 @@ my $ibx = {
        name => $group,
        version => $version,
        -primary_address => $addr,
+       indexlevel => 'basic',
 };
 $ibx = PublicInbox::Inbox->new($ibx);
 {
        local $ENV{HOME} = $home;
        my @cmd = ($init, $group, $mainrepo, 'http://example.com/', $addr);
-       push @cmd, "-V$version";
+       push @cmd, "-V$version", '-Lbasic';
        is(system(@cmd), 0, 'init OK');
        is(system(qw(git config), "--file=$home/.public-inbox/config",
                        "publicinbox.$group.newsgroup", $group),
@@ -71,8 +72,8 @@ $ibx = PublicInbox::Inbox->new($ibx);
        if ($version == 2) {
                $im = PublicInbox::V2Writable->new($ibx);
        } elsif ($version == 1) {
-               my $git = PublicInbox::Git->new($mainrepo);
-               $im = PublicInbox::Import->new($git, 'test', $addr);
+               use_ok 'PublicInbox::V1Writable';
+               $im = PublicInbox::V1Writable->new($ibx);
        } else {
                die "unsupported version: $version";
        }
@@ -261,6 +262,12 @@ EOF
                is($rdr, waitpid($rdr, 0), 'reader done');
                is($? >> 8, 0, 'no errors');
        }
+       SKIP: {
+               my @of = `lsof -p $pid 2>/dev/null`;
+               skip('lsof broken', 1) if (!scalar(@of) || $?);
+               my @xap = grep m!Search/Xapian!, @of;
+               is_deeply(\@xap, [], 'Xapian not loaded in nntpd');
+       }
        {
                setsockopt($s, IPPROTO_TCP, TCP_NODELAY, 1);
                syswrite($s, 'HDR List-id 1-');

diff --git a/t/over.t b/t/over.t
index 7a3c972104d6d15024b8fbc8c5e25465e182e308..c0d9d5e54e58a475026ffed20a3d71d0ad962d76 100644 (file)
--- a/t/over.t
+++ b/t/over.t
@@ -5,8 +5,7 @@ use warnings;
 use Test::More;
 use File::Temp qw/tempdir/;
 use Compress::Zlib qw(compress);
-# FIXME: allow using Over w/o Xapian
-foreach my $mod (qw(DBD::SQLite Search::Xapian)) {
+foreach my $mod (qw(DBD::SQLite)) {
        eval "require $mod";
        plan skip_all => "$mod missing for over.t" if $@;
 }

diff --git a/t/psgi_bad_mids.t b/t/psgi_bad_mids.t
index 5008f5be6b65aefec9b08595612e46126f4a9ad0..71eefe59971320b0fd60ccc769542da41eca192f 100644 (file)
--- a/t/psgi_bad_mids.t
+++ b/t/psgi_bad_mids.t
@@ -7,7 +7,7 @@ use File::Temp qw/tempdir/;
 use PublicInbox::MIME;
 use PublicInbox::Config;
 use PublicInbox::WWW;
-my @mods = qw(DBD::SQLite Search::Xapian HTTP::Request::Common Plack::Test
+my @mods = qw(DBD::SQLite HTTP::Request::Common Plack::Test
                URI::Escape Plack::Builder);
 foreach my $mod (@mods) {
        eval "require $mod";
@@ -22,6 +22,7 @@ my $ibx = {
        name => 'bad-mids',
        version => 2,
        -primary_address => 'test@example.com',
+       indexlevel => 'basic',
 };
 $ibx = PublicInbox::Inbox->new($ibx);
 my $im = PublicInbox::V2Writable->new($ibx, 1);

diff --git a/t/psgi_scan_all.t b/t/psgi_scan_all.t
index e9c439ec6b0e8e7bab0ec6afa41dbf4fe97ed921..2f54c8207578303bcc92f06facf880d19a388c4d 100644 (file)
--- a/t/psgi_scan_all.t
+++ b/t/psgi_scan_all.t
@@ -6,8 +6,7 @@ use Test::More;
 use Email::MIME;
 use File::Temp qw/tempdir/;
 use PublicInbox::Config;
-my @mods = qw(HTTP::Request::Common Plack::Test URI::Escape Search::Xapian
-       DBD::SQLite);
+my @mods = qw(HTTP::Request::Common Plack::Test URI::Escape DBD::SQLite);
 foreach my $mod (@mods) {
        eval "require $mod";
        plan skip_all => "$mod missing for psgi_scan_all.t" if $@;
@@ -26,6 +25,7 @@ foreach my $i (1..2) {
                mainrepo => $mainrepo,
                name => "test-$i",
                version => 2,
+               indexlevel => 'basic',
                -primary_address => $addr,
        };
        my $ibx = PublicInbox::Inbox->new($opt);

diff --git a/t/psgi_search.t b/t/psgi_search.t
index da6cc682004b33dadab7266adfbf20b25c090db8..1adc1d6e4e0eb7c8cb7384947feb0dddc6769ee9 100644 (file)
--- a/t/psgi_search.t
+++ b/t/psgi_search.t
@@ -8,13 +8,13 @@ use Email::MIME;
 use PublicInbox::Config;
 use PublicInbox::WWW;
 use bytes (); # only for bytes::length
-my @mods = qw(PublicInbox::SearchIdx HTTP::Request::Common Plack::Test
+my @mods = qw(Search::Xapian HTTP::Request::Common Plack::Test
                URI::Escape Plack::Builder);
 foreach my $mod (@mods) {
        eval "require $mod";
        plan skip_all => "$mod missing for psgi_search.t" if $@;
 }
-use_ok $_ foreach @mods;
+use_ok $_ foreach (@mods, qw(PublicInbox::SearchIdx));
 my $tmpdir = tempdir('pi-psgi-search.XXXXXX', TMPDIR => 1, CLEANUP => 1);
 my $git_dir = "$tmpdir/a.git";
 

diff --git a/t/purge.t b/t/purge.t
index 574935e08e72f1b01eb5700e6a07ba38952d0cb1..c1e0e9a023ab8e4b24d513ab7020591ac9ffa3a6 100644 (file)
--- a/t/purge.t
+++ b/t/purge.t
@@ -6,7 +6,7 @@ use Test::More;
 use File::Temp qw/tempdir/;
 require './t/common.perl';
 require_git(2.6);
-my @mods = qw(IPC::Run DBI DBD::SQLite Search::Xapian);
+my @mods = qw(IPC::Run DBI DBD::SQLite);
 foreach my $mod (@mods) {
        eval "require $mod";
        plan skip_all => "missing $mod for t/purge.t" if $@;

diff --git a/t/search-thr-index.t b/t/search-thr-index.t
index ed3e4e76278f7126459dfb3940cf280dea9d0b5e..848dc56459d5acf23c64dacf6f2726e359007b29 100644 (file)
--- a/t/search-thr-index.t
+++ b/t/search-thr-index.t
@@ -7,8 +7,9 @@ use Test::More;
 use File::Temp qw/tempdir/;
 use PublicInbox::MID qw(mids);
 use Email::MIME;
-eval { require PublicInbox::SearchIdx; };
-plan skip_all => "Xapian missing for search" if $@;
+eval { require Search::Xapian };
+plan skip_all => "Search::Xapian missing for search" if $@;
+require PublicInbox::SearchIdx;
 my $tmpdir = tempdir('pi-search-thr-index.XXXXXX', TMPDIR => 1, CLEANUP => 1);
 my $git_dir = "$tmpdir/a.git";
 

diff --git a/t/search.t b/t/search.t
index 493e00de5df17ff14cd64fda8e660f3b953fcfdc..a9d0c92030e1a936a5e7d5a77b4b22c65d5645c1 100644 (file)
--- a/t/search.t
+++ b/t/search.t
@@ -3,8 +3,9 @@
 use strict;
 use warnings;
 use Test::More;
-eval { require PublicInbox::SearchIdx; };
-plan skip_all => "Xapian missing for search" if $@;
+eval { require Search::Xapian };
+plan skip_all => "Search::Xapian missing for search" if $@;
+require PublicInbox::SearchIdx;
 use File::Temp qw/tempdir/;
 use Email::MIME;
 my $tmpdir = tempdir('pi-search-XXXXXX', TMPDIR => 1, CLEANUP => 1);
@@ -12,7 +13,7 @@ my $git_dir = "$tmpdir/a.git";
 my ($root_id, $last_id);
 
 is(0, system(qw(git init --shared -q --bare), $git_dir), "git init (main)");
-eval { PublicInbox::Search->new($git_dir) };
+eval { PublicInbox::Search->new($git_dir)->xdb };
 ok($@, "exception raised on non-existent DB");
 
 my $rw = PublicInbox::SearchIdx->new($git_dir, 1);

diff --git a/t/v1reindex.t b/t/v1reindex.t
index 33a36fad34b19791fcc20ef229e8ed3aa7d4c119..402ecd783ade4adf44a6ae0d62751189909eba99 100644 (file)
--- a/t/v1reindex.t
+++ b/t/v1reindex.t
@@ -209,8 +209,7 @@ ok(!-d $xap, 'Xapian directories removed again');
        delete $ibx->{mm};
        is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
-       my $mset = $ibx->search->reopen->query('hello world', {mset=>1});
-       is($mset->size, 0, "no Xapian search results");
+       isnt($ibx->search, 'no search for basic');
 
        my ($min, $max) = $ibx->mm->minmax;
        is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');

diff --git a/t/v2reindex.t b/t/v2reindex.t
index c416629c3f4155cc5bc0e5642f64a9b4952747fd..2a6fc555793d686119892c1a0082d1f07e39fbdd 100644 (file)
--- a/t/v2reindex.t
+++ b/t/v2reindex.t
@@ -225,8 +225,9 @@ ok(!-d $xap, 'Xapian directories removed again');
        delete $ibx->{mm};
        is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
-       my $mset = $ibx->search->query('freedom', {mset=>1});
-       is($mset->size, 0, "search fails on indexlevel='basic'");
+
+       isnt($ibx->search, 'no search for basic');
+
        for (<"$xap/*/*">) { $sizes{$ibx->{indexlevel}} += -s _ if -f $_ }
        ok($sizes{medium} > $sizes{basic}, 'basic is smaller than medium');
 

diff --git a/t/watch_maildir_v2.t b/t/watch_maildir_v2.t
index 5f968919dfff669080aa4af0909f5750d97965d0..b2cff4bac5b91439d60f5847a83a77d9374264a5 100644 (file)
--- a/t/watch_maildir_v2.t
+++ b/t/watch_maildir_v2.t
@@ -7,12 +7,12 @@ use Cwd;
 use PublicInbox::Config;
 require './t/common.perl';
 require_git(2.6);
-my @mods = qw(Filesys::Notify::Simple PublicInbox::V2Writable);
+my @mods = qw(Search::Xapian DBD::SQLite Filesys::Notify::Simple);
 foreach my $mod (@mods) {
        eval "require $mod";
        plan skip_all => "$mod missing for watch_maildir_v2.t" if $@;
 }
-
+require PublicInbox::V2Writable;
 my $tmpdir = tempdir('watch_maildir-v2-XXXXXX', TMPDIR => 1, CLEANUP => 1);
 my $mainrepo = "$tmpdir/v2";
 my $maildir = "$tmpdir/md";