xapcmd: new module for wrapping Xapian commands

Port public-inbox-compact(1) over to using it, and we will need
to wrap copydatabase(1) to ease glass migrations, too.

diff --git a/MANIFEST b/MANIFEST
index 2b101fa3ccbcefcd6a80a1575b5e4950195d9c50..dfc1f6607497acbc9acb55916c14a92aadb46583 100644 (file)
--- a/MANIFEST
+++ b/MANIFEST
@@ -139,6 +139,7 @@ lib/PublicInbox/WwwHighlight.pm
 lib/PublicInbox/WwwListing.pm
 lib/PublicInbox/WwwStream.pm
 lib/PublicInbox/WwwText.pm
+lib/PublicInbox/Xapcmd.pm
 sa_config/Makefile
 sa_config/README
 sa_config/root/etc/spamassassin/public-inbox.pre

diff --git a/lib/PublicInbox/Xapcmd.pm b/lib/PublicInbox/Xapcmd.pm
new file mode 100644 (file)
index 0000000..586d7e6
--- /dev/null
+++ b/lib/PublicInbox/Xapcmd.pm
@@ -0,0 +1,65 @@
+# Copyright (C) 2018-2019 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+package PublicInbox::Xapcmd;
+use strict;
+use warnings;
+use PublicInbox::Spawn qw(which spawn);
+use PublicInbox::Over;
+use File::Temp qw(tempdir);
+use File::Path qw(remove_tree);
+
+sub commit_changes ($$$) {
+       my ($im, $old, $new) = @_;
+       my @st = stat($old) or die "failed to stat($old): $!\n";
+
+       my $over = "$old/over.sqlite3";
+       if (-f $over) {
+               $over = PublicInbox::Over->new($over);
+               $over->connect->sqlite_backup_to_file("$new/over.sqlite3");
+       }
+       rename($old, "$new/old") or die "rename $old => $new/old: $!\n";
+       chmod($st[2] & 07777, $new) or die "chmod $old: $!\n";
+       rename($new, $old) or die "rename $new => $old: $!\n";
+       $im->lock_release;
+       remove_tree("$old/old") or die "failed to remove $old/old: $!\n";
+}
+
+sub run {
+       my ($ibx, $cmd) = @_;
+       my $dir = $ibx->{mainrepo} or die "no mainrepo in inbox\n";
+       which($cmd->[0]) or die "$cmd->[0] not found in PATH\n";
+       $ibx->umask_prepare;
+       my $old = $ibx->search->xdir(1);
+       -d $old or die "$old does not exist\n";
+       my $new = tempdir($cmd->[0].'-XXXXXXXX', CLEANUP => 1, DIR => $dir);
+       my $v = $ibx->{version} || 1;
+       my @cmds;
+       if ($v == 1) {
+               push @cmds, [@$cmd, $old, $new];
+       } else {
+               opendir my $dh, $old or die "Failed to opendir $old: $!\n";
+               while (defined(my $dn = readdir($dh))) {
+                       if ($dn =~ /\A\d+\z/) {
+                               push @cmds, [@$cmd, "$old/$dn", "$new/$dn"];
+                       } elsif ($dn eq '.' || $dn eq '..') {
+                       } elsif ($dn =~ /\Aover\.sqlite3/) {
+                       } else {
+                               warn "W: skipping unknown dir: $old/$dn\n"
+                       }
+               }
+               die "No Xapian parts found in $old\n" unless @cmds;
+       }
+       my $im = $ibx->importer(0);
+       $ibx->with_umask(sub {
+               $im->lock_acquire;
+               my %pids = map {; spawn($_) => join(' ', @$_) } @cmds;
+               while (scalar keys %pids) {
+                       my $pid = waitpid(-1, 0);
+                       my $desc = delete $pids{$pid};
+                       die "$desc failed: $?\n" if $?;
+               }
+               commit_changes($im, $old, $new);
+       });
+}
+
+1;

diff --git a/script/public-inbox-compact b/script/public-inbox-compact
index 395eec37a4cff439cc2d1db1afbac81e9fb2fa95..4aa6273dc2435272c537c2ebceb27b4eefbeef5a 100755 (executable)
--- a/script/public-inbox-compact
+++ b/script/public-inbox-compact
@@ -8,9 +8,7 @@ use PublicInbox::Search;
 use PublicInbox::Config;
 use PublicInbox::InboxWritable;
 use Cwd 'abs_path';
-use File::Temp qw(tempdir);
-use File::Path qw(remove_tree);
-use PublicInbox::Spawn qw(spawn);
+use PublicInbox::Xapcmd;
 my $usage = "Usage: public-inbox-compact REPO_DIR\n";
 my $dir = shift or die $usage;
 my $config = eval { PublicInbox::Config->new };
@@ -31,71 +29,7 @@ unless ($ibx) {
        };
        $ibx = PublicInbox::Inbox->new($ibx);
 }
-my $v = ($ibx->{version} || 1);
 $ibx = PublicInbox::InboxWritable->new($ibx);
-$ibx->umask_prepare;
-
-sub commit_changes ($$$) {
-       my ($im, $old, $new) = @_;
-       my @st = stat($old) or die "failed to stat($old): $!\n";
-
-       my $over = "$old/over.sqlite3";
-       if (-f $over) {
-               require PublicInbox::Over;
-               $over = PublicInbox::Over->new($over);
-               $over->connect->sqlite_backup_to_file("$new/over.sqlite3");
-       }
-       rename($old, "$new/old") or die "rename $old => $new/old: $!\n";
-       chmod($st[2] & 07777, $new) or die "chmod $old: $!\n";
-       rename($new, $old) or die "rename $new => $old: $!\n";
-       $im->lock_release;
-       remove_tree("$old/old") or die "failed to remove $old/old: $!\n";
-}
 
 # we rely on --no-renumber to keep docids synched to NNTP
-my @compact = qw(xapian-compact --no-renumber);
-if ($v == 2) {
-       require PublicInbox::V2Writable;
-       my $v2w = PublicInbox::V2Writable->new($ibx);
-       my $xap_v = 'xap'.PublicInbox::Search::SCHEMA_VERSION;
-       my $old = "$dir/$xap_v";
-       opendir my $dh, $old or die "Failed to opendir $old: $!\n";
-       my $new = tempdir('compact-XXXXXXXX', CLEANUP => 1, DIR => $dir);
-       $ibx->with_umask(sub {
-               $v2w->lock_acquire;
-               my %pids;
-               while (defined(my $dn = readdir($dh))) {
-                       if ($dn =~ /\A\d+\z/) {
-                               my $cmd = [ @compact, "$old/$dn", "$new/$dn" ];
-                               $pids{spawn($cmd)} = join(' ', @$cmd);
-                       } elsif ($dn eq '.' || $dn eq '..') {
-                       } elsif ($dn =~ /\Aover\.sqlite3/) {
-                       } else {
-                               warn "W: skipping unknown Xapian DB: $old/$dn\n"
-                       }
-               }
-               close $dh;
-               die "No Xapian parts found in $old\n" unless keys %pids;
-               while (scalar keys %pids) {
-                       my $pid = waitpid(-1, 0);
-                       my $desc = delete $pids{$pid};
-                       die "$desc failed: $?\n" if $?;
-               }
-               commit_changes($v2w, $old, $new);
-       });
-} elsif ($v == 1) {
-       require PublicInbox::Import;
-       my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
-       my $xap_v = 'xapian'.PublicInbox::Search::SCHEMA_VERSION;
-       my $v1_root = "$dir/public-inbox";
-       my $old = "$v1_root/$xap_v";
-       -d $old or die "$old does not exist\n";
-       my $new = tempdir('compact-XXXXXXXX', CLEANUP => 1, DIR => $v1_root);
-       $ibx->with_umask(sub {
-               $im->lock_acquire;
-               PublicInbox::Import::run_die([@compact, $old, $new]);
-               commit_changes($im, $old, $new);
-       });
-} else {
-       die "Unsupported inbox version: $v\n";
-}
+PublicInbox::Xapcmd::run($ibx, [qw(xapian-compact --no-renumber)]);