xcpdb: new tool which wraps Xapian's copydatabase(1)

copydatabase(1) is an existing Xapian tool which is the
recommended way to upgrade existing DBs to the latest Xapian
database format (currently "glass" for stable/released
versions).  Our use of Xapian relies on preserving document IDs,
so we'll wrap it like we do xapian-compact(1) and use the
"--no-renumber" switch.

I could not name the tool "public-inbox-copydatabase" since it
would be ambiguous as to which DB it's actually copying.  So, I
abbreviated the suffix to "xcpdb" (Xapian CoPy DataBase), which
I hope is acceptable and unambiguous.

diff --git a/Documentation/include.mk b/Documentation/include.mk
index 6415338b9158ff0ff5340e0728d598b4b51ea3dd..27d6ea6e41cc06dd9c4944e2b344a49fd49dfebc 100644 (file)
--- a/Documentation/include.mk
+++ b/Documentation/include.mk
@@ -26,11 +26,13 @@ podtext = $(PODTEXT) $(PODTEXT_OPTS)
 
 # MakeMaker only seems to support manpage sections 1 and 3...
 m1 =
-m1 += public-inbox-mda
+m1 += public-inbox-compact
 m1 += public-inbox-httpd
+m1 += public-inbox-index
+m1 += public-inbox-mda
 m1 += public-inbox-nntpd
 m1 += public-inbox-watch
-m1 += public-inbox-index
+m1 += public-inbox-xcpdb
 m5 =
 m5 += public-inbox-config
 m5 += public-inbox-v1-format

diff --git a/Documentation/public-inbox-xcpdb.pod b/Documentation/public-inbox-xcpdb.pod
new file mode 100644 (file)
index 0000000..4ff5186
--- /dev/null
+++ b/Documentation/public-inbox-xcpdb.pod
@@ -0,0 +1,51 @@
+=head1 NAME
+
+public-inbox-xcpdb - copy Xapian DBs (for format upgrades)
+
+=head1 SYNOPSIS
+
+       public-inbox-xcpdb INBOX_DIR
+
+=head1 DESCRIPTION
+
+public-inbox-xcpdb is a wrapper for L<copydatabase(1)> for
+upgrading to the latest database format supported by Xapian
+(e.g. "glass" or "honey").
+
+It locks the inbox and prevents other processes such as
+L<public-inbox-watch(1)> and L<public-inbox-mda(1)> from
+writing while it operates.
+
+This is intended for upgrading the database format used by
+Xapian.  It DOES NOT upgrade the schema used by the
+public-inbox search interface (see L<public-inbox-index(1)>).
+
+=head1 ENVIRONMENT
+
+=over 8
+
+=item PI_CONFIG
+
+The default config file, normally "~/.public-inbox/config".
+See L<public-inbox-config(5)>
+
+=back
+
+=head1 UPGRADING
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/>
+and L<http://hjrcffqmbrq6wope.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright 2019 all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+=head1 SEE ALSO
+
+L<copydatabase(1)>, L<public-inbox-index(1)>

diff --git a/MANIFEST b/MANIFEST
index dfc1f6607497acbc9acb55916c14a92aadb46583..efd5658df9b4b1a02ad62ec9d9674864a6008594 100644 (file)
--- a/MANIFEST
+++ b/MANIFEST
@@ -20,6 +20,7 @@ Documentation/public-inbox-overview.pod
 Documentation/public-inbox-v1-format.pod
 Documentation/public-inbox-v2-format.pod
 Documentation/public-inbox-watch.pod
+Documentation/public-inbox-xcpdb.pod
 Documentation/standards.perl
 Documentation/txt2pre
 HACKING
@@ -154,6 +155,7 @@ script/public-inbox-mda
 script/public-inbox-nntpd
 script/public-inbox-purge
 script/public-inbox-watch
+script/public-inbox-xcpdb
 script/public-inbox.cgi
 scripts/dc-dlvr
 scripts/dc-dlvr.pre

diff --git a/script/public-inbox-xcpdb b/script/public-inbox-xcpdb
new file mode 100755 (executable)
index 0000000..cbf9f55
--- /dev/null
+++ b/script/public-inbox-xcpdb
@@ -0,0 +1,18 @@
+#!/usr/bin/perl -w
+# Copyright (C) 2019 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+# xcpdb: Xapian copy database, a wrapper around Xapian's copydatabase(1)
+use PublicInbox::InboxWritable;
+use PublicInbox::Xapcmd;
+use PublicInbox::Admin;
+PublicInbox::Admin::require_or_die('-search');
+my $usage = "Usage: public-inbox-xcpdb INBOX_DIR\n";
+my @ibxs = PublicInbox::Admin::resolve_inboxes(\@ARGV) or die $usage;
+my $cmd = [qw(copydatabase --no-renumber)];
+open my $null, '>', '/dev/null' or die "failed to open /dev/null: $!\n";
+my $rdr = { 1 => fileno($null) };
+foreach (@ibxs) {
+       my $ibx = PublicInbox::InboxWritable->new($_);
+       # we rely on --no-renumber to keep docids synched to NNTP
+       PublicInbox::Xapcmd::run($ibx, $cmd, undef, $rdr);
+}

diff --git a/t/indexlevels-mirror.t b/t/indexlevels-mirror.t
index d124c7558ff790e49008e63d02d011e17c543db4..61053b669b75efda2d045f51e87c522e78eed25c 100644 (file)
--- a/t/indexlevels-mirror.t
+++ b/t/indexlevels-mirror.t
@@ -18,6 +18,7 @@ foreach my $mod (qw(DBD::SQLite)) {
 
 my $path = 'blib/script';
 my $index = "$path/public-inbox-index";
+my $xcpdb = "$path/public-inbox-xcpdb";
 
 my $mime = PublicInbox::MIME->create(
        header => [
@@ -108,6 +109,13 @@ sub import_index_incremental {
        ok($im->remove($mime), '2nd message removed');
        $im->done;
 
+       if ($level ne 'basic') {
+               is(system($xcpdb, $mirror), 0, "v$v xcpdb OK");
+               delete $ro_mirror->{$_} for (qw(over search));
+               ($nr, $msgs) = $ro_mirror->search->query('m:m@2');
+               is($nr, 1, "v$v found m\@2 via Xapian on $level");
+       }
+
        # sync the mirror
        is(system('git', "--git-dir=$fetch_dir", qw(fetch -q)), 0, 'fetch OK');
        is(system($index, $mirror), 0, "v$v index mirror again OK");
@@ -120,6 +128,10 @@ sub import_index_incremental {
                is_deeply([glob("$ibx->{mainrepo}/xap*/?/")], [],
                         'no Xapian partition directories for v2 basic');
        }
+       if ($level ne 'basic') {
+               ($nr, $msgs) = $ro_mirror->search->reopen->query('m:m@2');
+               is($nr, 0, "v$v m\@2 gone from Xapian in mirror on $level");
+       }
 }
 
 # we can probably cull some other tests and put full/medium tests, here
@@ -131,4 +143,14 @@ for my $level (qw(basic)) {
        }
 }
 
+SKIP: {
+       require PublicInbox::Search;
+       PublicInbox::Search::load_xapian() or skip 'Search::Xapian missing', 2;
+       for my $v (1..2) {
+               subtest("v$v indexlevel=medium" => sub {
+                       import_index_incremental($v, 'medium');
+               })
+       }
+}
+
 done_testing();