Documentation/include.mk | 6 ++++-- Documentation/public-inbox-xcpdb.pod | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ MANIFEST | 2 ++ script/public-inbox-xcpdb | 18 ++++++++++++++++++ t/indexlevels-mirror.t | 22 ++++++++++++++++++++++ diff --git a/Documentation/include.mk b/Documentation/include.mk index 6415338b9158ff0ff5340e0728d598b4b51ea3dd..27d6ea6e41cc06dd9c4944e2b344a49fd49dfebc 100644 --- a/Documentation/include.mk +++ b/Documentation/include.mk @@ -26,11 +26,13 @@ podtext = $(PODTEXT) $(PODTEXT_OPTS) # MakeMaker only seems to support manpage sections 1 and 3... m1 = -m1 += public-inbox-mda +m1 += public-inbox-compact m1 += public-inbox-httpd +m1 += public-inbox-index +m1 += public-inbox-mda m1 += public-inbox-nntpd m1 += public-inbox-watch -m1 += public-inbox-index +m1 += public-inbox-xcpdb m5 = m5 += public-inbox-config m5 += public-inbox-v1-format diff --git a/Documentation/public-inbox-xcpdb.pod b/Documentation/public-inbox-xcpdb.pod new file mode 100644 index 0000000000000000000000000000000000000000..4ff51867367cb9ab1ce920fedf1b18e39ae19d18 --- /dev/null +++ b/Documentation/public-inbox-xcpdb.pod @@ -0,0 +1,51 @@ +=head1 NAME + +public-inbox-xcpdb - copy Xapian DBs (for format upgrades) + +=head1 SYNOPSIS + + public-inbox-xcpdb INBOX_DIR + +=head1 DESCRIPTION + +public-inbox-xcpdb is a wrapper for L for +upgrading to the latest database format supported by Xapian +(e.g. "glass" or "honey"). + +It locks the inbox and prevents other processes such as +L and L from +writing while it operates. + +This is intended for upgrading the database format used by +Xapian. It DOES NOT upgrade the schema used by the +public-inbox search interface (see L). + +=head1 ENVIRONMENT + +=over 8 + +=item PI_CONFIG + +The default config file, normally "~/.public-inbox/config". +See L + +=back + +=head1 UPGRADING + +=head1 CONTACT + +Feedback welcome via plain-text mail to L + +The mail archives are hosted at L +and L + +=head1 COPYRIGHT + +Copyright 2019 all contributors L + +License: AGPL-3.0+ L + +=head1 SEE ALSO + +L, L diff --git a/MANIFEST b/MANIFEST index dfc1f6607497acbc9acb55916c14a92aadb46583..efd5658df9b4b1a02ad62ec9d9674864a6008594 100644 --- a/MANIFEST +++ b/MANIFEST @@ -20,6 +20,7 @@ Documentation/public-inbox-overview.pod Documentation/public-inbox-v1-format.pod Documentation/public-inbox-v2-format.pod Documentation/public-inbox-watch.pod +Documentation/public-inbox-xcpdb.pod Documentation/standards.perl Documentation/txt2pre HACKING @@ -154,6 +155,7 @@ script/public-inbox-mda script/public-inbox-nntpd script/public-inbox-purge script/public-inbox-watch +script/public-inbox-xcpdb script/public-inbox.cgi scripts/dc-dlvr scripts/dc-dlvr.pre diff --git a/script/public-inbox-xcpdb b/script/public-inbox-xcpdb new file mode 100755 index 0000000000000000000000000000000000000000..cbf9f556a554b944d38bb1b42ec6e2a7d002efa4 --- /dev/null +++ b/script/public-inbox-xcpdb @@ -0,0 +1,18 @@ +#!/usr/bin/perl -w +# Copyright (C) 2019 all contributors +# License: AGPL-3.0+ +# xcpdb: Xapian copy database, a wrapper around Xapian's copydatabase(1) +use PublicInbox::InboxWritable; +use PublicInbox::Xapcmd; +use PublicInbox::Admin; +PublicInbox::Admin::require_or_die('-search'); +my $usage = "Usage: public-inbox-xcpdb INBOX_DIR\n"; +my @ibxs = PublicInbox::Admin::resolve_inboxes(\@ARGV) or die $usage; +my $cmd = [qw(copydatabase --no-renumber)]; +open my $null, '>', '/dev/null' or die "failed to open /dev/null: $!\n"; +my $rdr = { 1 => fileno($null) }; +foreach (@ibxs) { + my $ibx = PublicInbox::InboxWritable->new($_); + # we rely on --no-renumber to keep docids synched to NNTP + PublicInbox::Xapcmd::run($ibx, $cmd, undef, $rdr); +} diff --git a/t/indexlevels-mirror.t b/t/indexlevels-mirror.t index d124c7558ff790e49008e63d02d011e17c543db4..61053b669b75efda2d045f51e87c522e78eed25c 100644 --- a/t/indexlevels-mirror.t +++ b/t/indexlevels-mirror.t @@ -18,6 +18,7 @@ } my $path = 'blib/script'; my $index = "$path/public-inbox-index"; +my $xcpdb = "$path/public-inbox-xcpdb"; my $mime = PublicInbox::MIME->create( header => [ @@ -108,6 +109,13 @@ # remove message from master ok($im->remove($mime), '2nd message removed'); $im->done; + if ($level ne 'basic') { + is(system($xcpdb, $mirror), 0, "v$v xcpdb OK"); + delete $ro_mirror->{$_} for (qw(over search)); + ($nr, $msgs) = $ro_mirror->search->query('m:m@2'); + is($nr, 1, "v$v found m\@2 via Xapian on $level"); + } + # sync the mirror is(system('git', "--git-dir=$fetch_dir", qw(fetch -q)), 0, 'fetch OK'); is(system($index, $mirror), 0, "v$v index mirror again OK"); @@ -119,6 +127,10 @@ if ($v == 2 && $level eq 'basic') { is_deeply([glob("$ibx->{mainrepo}/xap*/?/")], [], 'no Xapian partition directories for v2 basic'); + } + if ($level ne 'basic') { + ($nr, $msgs) = $ro_mirror->search->reopen->query('m:m@2'); + is($nr, 0, "v$v m\@2 gone from Xapian in mirror on $level"); } } @@ -127,6 +139,16 @@ for my $level (qw(basic)) { for my $v (1..2) { subtest("v$v indexlevel=$level" => sub { import_index_incremental($v, $level); + }) + } +} + +SKIP: { + require PublicInbox::Search; + PublicInbox::Search::load_xapian() or skip 'Search::Xapian missing', 2; + for my $v (1..2) { + subtest("v$v indexlevel=medium" => sub { + import_index_incremental($v, 'medium'); }) } }