xcpdb: implement progress reporting

Copying an entire Xapian DB is horribly slow whether it's done
via Perl or copydatabase(1).  So displaying some progress
indication is good for user experience.

While we're at it, prefix xapian-compact output, too; since
parallel processes end up clobbering each other.

diff --git a/lib/PublicInbox/Xapcmd.pm b/lib/PublicInbox/Xapcmd.pm
index 4555340a6d7e8b2fd0eed01a4258074c07c5a167..99f0e7c13d9e8d07927e7e56de3833699025abf4 100644 (file)
--- a/lib/PublicInbox/Xapcmd.pm
+++ b/lib/PublicInbox/Xapcmd.pm
@@ -82,9 +82,21 @@ sub prepare_reindex ($$) {
        }
 }
 
+sub progress_prepare ($) {
+       my ($opt) = @_;
+       if ($opt->{quiet}) {
+               open my $null, '>', '/dev/null' or
+                       die "failed to open /dev/null: $!\n";
+               $opt->{1} = fileno($null);
+               $opt->{-dev_null} = $null;
+       } else {
+               $opt->{-progress} = 1;
+       }
+}
+
 sub run {
        my ($ibx, $cmd, $env, $opt) = @_;
-       $opt ||= {};
+       progress_prepare($opt ||= {});
        my $dir = $ibx->{mainrepo} or die "no mainrepo in inbox\n";
        my $exe = $cmd->[0];
        my $pfx = $exe;
@@ -161,6 +173,8 @@ sub cpdb_retryable ($$) {
        0;
 }
 
+# Like copydatabase(1), this is horribly slow; and it doesn't seem due
+# to the overhead of Perl.
 sub cpdb {
        my ($args, $env, $opt) = @_;
        my ($old, $new) = @$args;
@@ -172,6 +186,7 @@ sub cpdb {
        my $creat = Search::Xapian::DB_CREATE();
        my $dst = Search::Xapian::WritableDatabase->new($tmp, $creat);
        my ($it, $end);
+       my ($pfx, $nr, $tot, $fmt); # progress output
 
        do {
                eval {
@@ -181,6 +196,13 @@ sub cpdb {
 
                        $it = $src->postlist_begin('');
                        $end = $src->postlist_end('');
+                       if ($opt->{-progress}) {
+                               $nr = 0;
+                               $pfx = (split('/', $old))[-1].':';
+                               $tot = $src->get_doccount;
+                               $fmt = "$pfx % ".length($tot)."u/$tot\n";
+                               warn "$pfx copying $tot documents\n";
+                       }
                };
        } while (cpdb_retryable($src, $@));
 
@@ -191,6 +213,9 @@ sub cpdb {
                                my $doc = $src->get_document($docid);
                                $dst->replace_document($docid, $doc);
                                $it->inc;
+                               if ($fmt && !(++$nr & 1023)) {
+                                       warn(sprintf($fmt, $nr));
+                               }
                        }
 
                        # unlike copydatabase(1), we don't copy spelling
@@ -200,10 +225,12 @@ sub cpdb {
                };
        } while (cpdb_retryable($src, $@));
 
+       warn(sprintf($fmt, $nr)) if $fmt;
        return unless $opt->{compact};
 
        $src = $dst = undef; # flushes and closes
 
+       warn "$pfx compacting...\n" if $pfx;
        # this is probably the best place to do xapian-compact
        # since $dst isn't readable by HTTP or NNTP clients, yet:
        my $cmd = [ $XAPIAN_COMPACT, '--no-renumber', $tmp, $new ];
@@ -212,10 +239,22 @@ sub cpdb {
                defined(my $dst = $opt->{$fd}) or next;
                $rdr->{$fd} = $dst;
        }
+
+       my ($r, $w);
+       if ($pfx && pipe($r, $w)) {
+               $rdr->{1} = fileno($w);
+       }
        my $pid = spawn($cmd, $env, $rdr);
-       my $r = waitpid($pid, 0);
-       if ($? || $r != $pid) {
-               die join(' ', @$cmd)." failed: $? (pid=$pid, reaped=$r)\n";
+       if ($pfx) {
+               close $w or die "close: \$w: $!";
+               foreach (<$r>) {
+                       s/\r/\r$pfx /g;
+                       warn "$pfx $_";
+               }
+       }
+       my $rp = waitpid($pid, 0);
+       if ($? || $rp != $pid) {
+               die join(' ', @$cmd)." failed: $? (pid=$pid, reaped=$rp)\n";
        }
        remove_tree($tmp) or die "failed to remove $tmp: $!\n";
 }

diff --git a/script/public-inbox-xcpdb b/script/public-inbox-xcpdb
index 78d37da266153465878902475de3a40f0fd26521..5b66337b47a10870543e599c80516bf84407c32f 100755 (executable)
--- a/script/public-inbox-xcpdb
+++ b/script/public-inbox-xcpdb
@@ -9,12 +9,9 @@ use PublicInbox::Admin;
 PublicInbox::Admin::require_or_die('-search');
 my $usage = "Usage: public-inbox-xcpdb INBOX_DIR\n";
 my $opt = {};
-GetOptions($opt, qw(compact)) or die "bad command-line args\n$usage";
+GetOptions($opt, qw(compact quiet|q)) or die "bad command-line args\n$usage";
 my @ibxs = PublicInbox::Admin::resolve_inboxes(\@ARGV) or die $usage;
-
 my $cmd = [ \&PublicInbox::Xapcmd::cpdb ];
-open my $null, '>', '/dev/null' or die "failed to open /dev/null: $!\n";
-$opt->{1} = fileno($null);
 foreach (@ibxs) {
        my $ibx = PublicInbox::InboxWritable->new($_);
        # we rely on --no-renumber to keep docids synched to NNTP

diff --git a/t/indexlevels-mirror.t b/t/indexlevels-mirror.t
index 61053b669b75efda2d045f51e87c522e78eed25c..57a776f7c84f2c467c340b6456581907d805eb9f 100644 (file)
--- a/t/indexlevels-mirror.t
+++ b/t/indexlevels-mirror.t
@@ -18,7 +18,7 @@ foreach my $mod (qw(DBD::SQLite)) {
 
 my $path = 'blib/script';
 my $index = "$path/public-inbox-index";
-my $xcpdb = "$path/public-inbox-xcpdb";
+my @xcpdb = ("$path/public-inbox-xcpdb", '-q');
 
 my $mime = PublicInbox::MIME->create(
        header => [
@@ -110,7 +110,7 @@ sub import_index_incremental {
        $im->done;
 
        if ($level ne 'basic') {
-               is(system($xcpdb, $mirror), 0, "v$v xcpdb OK");
+               is(system(@xcpdb, $mirror), 0, "v$v xcpdb OK");
                delete $ro_mirror->{$_} for (qw(over search));
                ($nr, $msgs) = $ro_mirror->search->query('m:m@2');
                is($nr, 1, "v$v found m\@2 via Xapian on $level");