Copying an entire Xapian DB is horribly slow whether it's done
via Perl or copydatabase(1). So displaying some progress
indication is good for user experience.
While we're at it, prefix xapian-compact output, too; since
parallel processes end up clobbering each other.
+sub progress_prepare ($) {
+ my ($opt) = @_;
+ if ($opt->{quiet}) {
+ open my $null, '>', '/dev/null' or
+ die "failed to open /dev/null: $!\n";
+ $opt->{1} = fileno($null);
+ $opt->{-dev_null} = $null;
+ } else {
+ $opt->{-progress} = 1;
+ }
+}
+
sub run {
my ($ibx, $cmd, $env, $opt) = @_;
sub run {
my ($ibx, $cmd, $env, $opt) = @_;
+ progress_prepare($opt ||= {});
my $dir = $ibx->{mainrepo} or die "no mainrepo in inbox\n";
my $exe = $cmd->[0];
my $pfx = $exe;
my $dir = $ibx->{mainrepo} or die "no mainrepo in inbox\n";
my $exe = $cmd->[0];
my $pfx = $exe;
+# Like copydatabase(1), this is horribly slow; and it doesn't seem due
+# to the overhead of Perl.
sub cpdb {
my ($args, $env, $opt) = @_;
my ($old, $new) = @$args;
sub cpdb {
my ($args, $env, $opt) = @_;
my ($old, $new) = @$args;
my $creat = Search::Xapian::DB_CREATE();
my $dst = Search::Xapian::WritableDatabase->new($tmp, $creat);
my ($it, $end);
my $creat = Search::Xapian::DB_CREATE();
my $dst = Search::Xapian::WritableDatabase->new($tmp, $creat);
my ($it, $end);
+ my ($pfx, $nr, $tot, $fmt); # progress output
$it = $src->postlist_begin('');
$end = $src->postlist_end('');
$it = $src->postlist_begin('');
$end = $src->postlist_end('');
+ if ($opt->{-progress}) {
+ $nr = 0;
+ $pfx = (split('/', $old))[-1].':';
+ $tot = $src->get_doccount;
+ $fmt = "$pfx % ".length($tot)."u/$tot\n";
+ warn "$pfx copying $tot documents\n";
+ }
};
} while (cpdb_retryable($src, $@));
};
} while (cpdb_retryable($src, $@));
my $doc = $src->get_document($docid);
$dst->replace_document($docid, $doc);
$it->inc;
my $doc = $src->get_document($docid);
$dst->replace_document($docid, $doc);
$it->inc;
+ if ($fmt && !(++$nr & 1023)) {
+ warn(sprintf($fmt, $nr));
+ }
}
# unlike copydatabase(1), we don't copy spelling
}
# unlike copydatabase(1), we don't copy spelling
};
} while (cpdb_retryable($src, $@));
};
} while (cpdb_retryable($src, $@));
+ warn(sprintf($fmt, $nr)) if $fmt;
return unless $opt->{compact};
$src = $dst = undef; # flushes and closes
return unless $opt->{compact};
$src = $dst = undef; # flushes and closes
+ warn "$pfx compacting...\n" if $pfx;
# this is probably the best place to do xapian-compact
# since $dst isn't readable by HTTP or NNTP clients, yet:
my $cmd = [ $XAPIAN_COMPACT, '--no-renumber', $tmp, $new ];
# this is probably the best place to do xapian-compact
# since $dst isn't readable by HTTP or NNTP clients, yet:
my $cmd = [ $XAPIAN_COMPACT, '--no-renumber', $tmp, $new ];
defined(my $dst = $opt->{$fd}) or next;
$rdr->{$fd} = $dst;
}
defined(my $dst = $opt->{$fd}) or next;
$rdr->{$fd} = $dst;
}
+
+ my ($r, $w);
+ if ($pfx && pipe($r, $w)) {
+ $rdr->{1} = fileno($w);
+ }
my $pid = spawn($cmd, $env, $rdr);
my $pid = spawn($cmd, $env, $rdr);
- my $r = waitpid($pid, 0);
- if ($? || $r != $pid) {
- die join(' ', @$cmd)." failed: $? (pid=$pid, reaped=$r)\n";
+ if ($pfx) {
+ close $w or die "close: \$w: $!";
+ foreach (<$r>) {
+ s/\r/\r$pfx /g;
+ warn "$pfx $_";
+ }
+ }
+ my $rp = waitpid($pid, 0);
+ if ($? || $rp != $pid) {
+ die join(' ', @$cmd)." failed: $? (pid=$pid, reaped=$rp)\n";
}
remove_tree($tmp) or die "failed to remove $tmp: $!\n";
}
}
remove_tree($tmp) or die "failed to remove $tmp: $!\n";
}
PublicInbox::Admin::require_or_die('-search');
my $usage = "Usage: public-inbox-xcpdb INBOX_DIR\n";
my $opt = {};
PublicInbox::Admin::require_or_die('-search');
my $usage = "Usage: public-inbox-xcpdb INBOX_DIR\n";
my $opt = {};
-GetOptions($opt, qw(compact)) or die "bad command-line args\n$usage";
+GetOptions($opt, qw(compact quiet|q)) or die "bad command-line args\n$usage";
my @ibxs = PublicInbox::Admin::resolve_inboxes(\@ARGV) or die $usage;
my @ibxs = PublicInbox::Admin::resolve_inboxes(\@ARGV) or die $usage;
my $cmd = [ \&PublicInbox::Xapcmd::cpdb ];
my $cmd = [ \&PublicInbox::Xapcmd::cpdb ];
-open my $null, '>', '/dev/null' or die "failed to open /dev/null: $!\n";
-$opt->{1} = fileno($null);
foreach (@ibxs) {
my $ibx = PublicInbox::InboxWritable->new($_);
# we rely on --no-renumber to keep docids synched to NNTP
foreach (@ibxs) {
my $ibx = PublicInbox::InboxWritable->new($_);
# we rely on --no-renumber to keep docids synched to NNTP
my $path = 'blib/script';
my $index = "$path/public-inbox-index";
my $path = 'blib/script';
my $index = "$path/public-inbox-index";
-my $xcpdb = "$path/public-inbox-xcpdb";
+my @xcpdb = ("$path/public-inbox-xcpdb", '-q');
my $mime = PublicInbox::MIME->create(
header => [
my $mime = PublicInbox::MIME->create(
header => [
$im->done;
if ($level ne 'basic') {
$im->done;
if ($level ne 'basic') {
- is(system($xcpdb, $mirror), 0, "v$v xcpdb OK");
+ is(system(@xcpdb, $mirror), 0, "v$v xcpdb OK");
delete $ro_mirror->{$_} for (qw(over search));
($nr, $msgs) = $ro_mirror->search->query('m:m@2');
is($nr, 1, "v$v found m\@2 via Xapian on $level");
delete $ro_mirror->{$_} for (qw(over search));
($nr, $msgs) = $ro_mirror->search->query('m:m@2');
is($nr, 1, "v$v found m\@2 via Xapian on $level");