From ac5a695030be64059fcb1ae826a0f3bfd2ca035b Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Thu, 23 May 2019 09:36:46 +0000 Subject: [PATCH] xcpdb: new tool which wraps Xapian's copydatabase(1) copydatabase(1) is an existing Xapian tool which is the recommended way to upgrade existing DBs to the latest Xapian database format (currently "glass" for stable/released versions). Our use of Xapian relies on preserving document IDs, so we'll wrap it like we do xapian-compact(1) and use the "--no-renumber" switch. I could not name the tool "public-inbox-copydatabase" since it would be ambiguous as to which DB it's actually copying. So, I abbreviated the suffix to "xcpdb" (Xapian CoPy DataBase), which I hope is acceptable and unambiguous. --- Documentation/include.mk | 6 ++-- Documentation/public-inbox-xcpdb.pod | 51 ++++++++++++++++++++++++++++ MANIFEST | 2 ++ script/public-inbox-xcpdb | 18 ++++++++++ t/indexlevels-mirror.t | 22 ++++++++++++ 5 files changed, 97 insertions(+), 2 deletions(-) create mode 100644 Documentation/public-inbox-xcpdb.pod create mode 100755 script/public-inbox-xcpdb diff --git a/Documentation/include.mk b/Documentation/include.mk index 6415338b..27d6ea6e 100644 --- a/Documentation/include.mk +++ b/Documentation/include.mk @@ -26,11 +26,13 @@ podtext = $(PODTEXT) $(PODTEXT_OPTS) # MakeMaker only seems to support manpage sections 1 and 3... m1 = -m1 += public-inbox-mda +m1 += public-inbox-compact m1 += public-inbox-httpd +m1 += public-inbox-index +m1 += public-inbox-mda m1 += public-inbox-nntpd m1 += public-inbox-watch -m1 += public-inbox-index +m1 += public-inbox-xcpdb m5 = m5 += public-inbox-config m5 += public-inbox-v1-format diff --git a/Documentation/public-inbox-xcpdb.pod b/Documentation/public-inbox-xcpdb.pod new file mode 100644 index 00000000..4ff51867 --- /dev/null +++ b/Documentation/public-inbox-xcpdb.pod @@ -0,0 +1,51 @@ +=head1 NAME + +public-inbox-xcpdb - copy Xapian DBs (for format upgrades) + +=head1 SYNOPSIS + + public-inbox-xcpdb INBOX_DIR + +=head1 DESCRIPTION + +public-inbox-xcpdb is a wrapper for L for +upgrading to the latest database format supported by Xapian +(e.g. "glass" or "honey"). + +It locks the inbox and prevents other processes such as +L and L from +writing while it operates. + +This is intended for upgrading the database format used by +Xapian. It DOES NOT upgrade the schema used by the +public-inbox search interface (see L). + +=head1 ENVIRONMENT + +=over 8 + +=item PI_CONFIG + +The default config file, normally "~/.public-inbox/config". +See L + +=back + +=head1 UPGRADING + +=head1 CONTACT + +Feedback welcome via plain-text mail to L + +The mail archives are hosted at L +and L + +=head1 COPYRIGHT + +Copyright 2019 all contributors L + +License: AGPL-3.0+ L + +=head1 SEE ALSO + +L, L diff --git a/MANIFEST b/MANIFEST index dfc1f660..efd5658d 100644 --- a/MANIFEST +++ b/MANIFEST @@ -20,6 +20,7 @@ Documentation/public-inbox-overview.pod Documentation/public-inbox-v1-format.pod Documentation/public-inbox-v2-format.pod Documentation/public-inbox-watch.pod +Documentation/public-inbox-xcpdb.pod Documentation/standards.perl Documentation/txt2pre HACKING @@ -154,6 +155,7 @@ script/public-inbox-mda script/public-inbox-nntpd script/public-inbox-purge script/public-inbox-watch +script/public-inbox-xcpdb script/public-inbox.cgi scripts/dc-dlvr scripts/dc-dlvr.pre diff --git a/script/public-inbox-xcpdb b/script/public-inbox-xcpdb new file mode 100755 index 00000000..cbf9f556 --- /dev/null +++ b/script/public-inbox-xcpdb @@ -0,0 +1,18 @@ +#!/usr/bin/perl -w +# Copyright (C) 2019 all contributors +# License: AGPL-3.0+ +# xcpdb: Xapian copy database, a wrapper around Xapian's copydatabase(1) +use PublicInbox::InboxWritable; +use PublicInbox::Xapcmd; +use PublicInbox::Admin; +PublicInbox::Admin::require_or_die('-search'); +my $usage = "Usage: public-inbox-xcpdb INBOX_DIR\n"; +my @ibxs = PublicInbox::Admin::resolve_inboxes(\@ARGV) or die $usage; +my $cmd = [qw(copydatabase --no-renumber)]; +open my $null, '>', '/dev/null' or die "failed to open /dev/null: $!\n"; +my $rdr = { 1 => fileno($null) }; +foreach (@ibxs) { + my $ibx = PublicInbox::InboxWritable->new($_); + # we rely on --no-renumber to keep docids synched to NNTP + PublicInbox::Xapcmd::run($ibx, $cmd, undef, $rdr); +} diff --git a/t/indexlevels-mirror.t b/t/indexlevels-mirror.t index d124c755..61053b66 100644 --- a/t/indexlevels-mirror.t +++ b/t/indexlevels-mirror.t @@ -18,6 +18,7 @@ foreach my $mod (qw(DBD::SQLite)) { my $path = 'blib/script'; my $index = "$path/public-inbox-index"; +my $xcpdb = "$path/public-inbox-xcpdb"; my $mime = PublicInbox::MIME->create( header => [ @@ -108,6 +109,13 @@ sub import_index_incremental { ok($im->remove($mime), '2nd message removed'); $im->done; + if ($level ne 'basic') { + is(system($xcpdb, $mirror), 0, "v$v xcpdb OK"); + delete $ro_mirror->{$_} for (qw(over search)); + ($nr, $msgs) = $ro_mirror->search->query('m:m@2'); + is($nr, 1, "v$v found m\@2 via Xapian on $level"); + } + # sync the mirror is(system('git', "--git-dir=$fetch_dir", qw(fetch -q)), 0, 'fetch OK'); is(system($index, $mirror), 0, "v$v index mirror again OK"); @@ -120,6 +128,10 @@ sub import_index_incremental { is_deeply([glob("$ibx->{mainrepo}/xap*/?/")], [], 'no Xapian partition directories for v2 basic'); } + if ($level ne 'basic') { + ($nr, $msgs) = $ro_mirror->search->reopen->query('m:m@2'); + is($nr, 0, "v$v m\@2 gone from Xapian in mirror on $level"); + } } # we can probably cull some other tests and put full/medium tests, here @@ -131,4 +143,14 @@ for my $level (qw(basic)) { } } +SKIP: { + require PublicInbox::Search; + PublicInbox::Search::load_xapian() or skip 'Search::Xapian missing', 2; + for my $v (1..2) { + subtest("v$v indexlevel=medium" => sub { + import_index_incremental($v, 'medium'); + }) + } +} + done_testing(); -- 2.44.0