X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=script%2Fpublic-inbox-xcpdb;h=24fc5a252b4e7e6415b0521c29472319e8218376;hb=4eee5af6011cc8cdefb66c9729952c7eff5c0b0b;hp=ef64e58f54675a30091f10befe1e7cd4321bd32a;hpb=69d84811ba0c4c60a5fb6893817328d76444cdf5;p=public-inbox.git

diff --git a/script/public-inbox-xcpdb b/script/public-inbox-xcpdb
index ef64e58f..24fc5a25 100755
--- a/script/public-inbox-xcpdb
+++ b/script/public-inbox-xcpdb
@@ -1,19 +1,68 @@
-#!/usr/bin/perl -w
-# Copyright (C) 2019 all contributors <meta@public-inbox.org>
+#!perl -w
+# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-# xcpdb: Xapian copy database, a wrapper around Xapian's copydatabase(1)
+use strict;
+use v5.10.1;
 use Getopt::Long qw(:config gnu_getopt no_ignore_case auto_abbrev);
-use PublicInbox::InboxWritable;
-use PublicInbox::Xapcmd;
+my $help = <<EOF; # the following should fit w/o scrolling in 80x24 term:
+usage: public-inbox-xcpdb [options] <INBOX_DIR|EXTINDEX_DIR>
+
+  upgrade or reshard Xapian DB(s) used by public-inbox
+
+options:
+
+  --compact | -c      run public-inbox-compact(1) after indexing
+  --all               copy all configured inboxes
+  --reshard=NUM       change number the number of shards
+  --jobs=NUM          limit parallelism to JOBS count
+  --verbose | -v      increase verbosity (may be repeated)
+  --sequential-shard  copy+index Xapian shards sequentially (for slow HDD)
+
+index options (see public-inbox-index(1) man page for full description):
+
+  --no-fsync          speed up indexing, risk corruption on power outage
+  --batch-size=BYTES  flush changes to OS after a given number of bytes
+  --max-size=BYTES    do not index messages larger than the given size
+
+See public-inbox-xcpdb(1) man page for full documentation.
+EOF
+my $opt = { quiet => -1, compact => 0, fsync => 1, -eidx_ok => 1 };
+GetOptions($opt, qw(
+	fsync|sync! compact|c reshard|R=i
+	max_size|max-size=s batch_size|batch-size=s
+	sequential-shard|seq-shard
+	jobs|j=i quiet|q verbose|v
+	blocksize|b=s no-full|n fuller|F
+	all C=s@ help|h)) or die $help;
+if ($opt->{help}) { print $help; exit 0 };
+
 use PublicInbox::Admin;
 PublicInbox::Admin::require_or_die('-search');
-my $usage = "Usage: public-inbox-xcpdb [--compact] INBOX_DIR\n";
-my $opt = {};
-GetOptions($opt, qw(compact), @PublicInbox::Xapcmd::COMPACT_OPT) or
-	die "bad command-line args\n$usage";
-my @ibxs = PublicInbox::Admin::resolve_inboxes(\@ARGV) or die $usage;
-foreach (@ibxs) {
-	my $ibx = PublicInbox::InboxWritable->new($_);
-	# we rely on --no-renumber to keep docids synched to NNTP
+PublicInbox::Admin::do_chdir(delete $opt->{C});
+
+require PublicInbox::Config;
+my $cfg = PublicInbox::Config->new;
+my ($ibxs, $eidxs) = PublicInbox::Admin::resolve_inboxes(\@ARGV, $opt, $cfg);
+unless ($ibxs) { print STDERR $help; exit 1 }
+my $idx_env = PublicInbox::Admin::index_prepare($opt, $cfg);
+
+# we only set XAPIAN_FLUSH_THRESHOLD for index, since cpdb doesn't
+# know sizes, only doccounts
+$opt->{-idx_env} = $idx_env;
+
+if ($opt->{'sequential-shard'} && ($opt->{jobs} // 1) > 1) {
+	warn "W: --jobs=$opt->{jobs} ignored with --sequential-shard\n";
+	$opt->{jobs} = 0;
+}
+
+require PublicInbox::InboxWritable;
+require PublicInbox::Xapcmd;
+# we rely on --no-renumber to keep docids synched for NNTP(artnum) + IMAP(UID)
+for my $ibx (@$ibxs) {
+	$ibx = PublicInbox::InboxWritable->new($ibx);
 	PublicInbox::Xapcmd::run($ibx, 'cpdb', $opt);
 }
+
+for my $eidx (@$eidxs) {
+	PublicInbox::Xapcmd::run($eidx, 'cpdb', $opt);
+}