X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FXapcmd.pm;h=6a74daf92b5be1e4345ce473f28a66b3db5e7f1b;hb=1814b1a0b78770c8ba9e7a0adef56c4c324d4064;hp=c807bf1014aef78d63f814ed17b4d7ead2840b00;hpb=6466b21e0776fdd88648730e7248d6887d380224;p=public-inbox.git diff --git a/lib/PublicInbox/Xapcmd.pm b/lib/PublicInbox/Xapcmd.pm index c807bf10..6a74daf9 100644 --- a/lib/PublicInbox/Xapcmd.pm +++ b/lib/PublicInbox/Xapcmd.pm @@ -1,14 +1,15 @@ -# Copyright (C) 2018-2019 all contributors +# Copyright (C) 2018-2020 all contributors # License: AGPL-3.0+ package PublicInbox::Xapcmd; use strict; -use warnings; -use PublicInbox::Spawn qw(which spawn); +use PublicInbox::Spawn qw(which popen_rd nodatacow_dir); +use PublicInbox::Admin qw(setup_signals); use PublicInbox::Over; -use PublicInbox::Search; -use File::Temp qw(tempdir); +use PublicInbox::SearchIdx; +use File::Temp 0.19 (); # ->newdir use File::Path qw(remove_tree); use File::Basename qw(dirname); +use POSIX qw(WNOHANG); # support testing with dev versions of Xapian which installs # commands with a version number suffix (e.g. "xapian-compact-1.5") @@ -18,25 +19,27 @@ our @COMPACT_OPT = qw(jobs|j=i quiet|q blocksize|b=s no-full|n fuller|F); sub commit_changes ($$$$) { my ($ibx, $im, $tmp, $opt) = @_; my $reshard = $opt->{reshard}; - my $reindex = $opt->{reindex}; $SIG{INT} or die 'BUG: $SIG{INT} not handled'; my @old_shard; + my $over_chg; - while (my ($old, $new) = each %$tmp) { + while (my ($old, $newdir) = each %$tmp) { next if $old eq ''; # no invalid paths my @st = stat($old); if (!@st && !defined($opt->{reshard})) { die "failed to stat($old): $!"; } + my $new = $newdir->dirname if defined($newdir); my $over = "$old/over.sqlite3"; if (-f $over) { # only for v1, v2 over is untouched defined $new or die "BUG: $over exists when culling v2"; $over = PublicInbox::Over->new($over); my $tmp_over = "$new/over.sqlite3"; - $over->connect->sqlite_backup_to_file($tmp_over); + $over->dbh->sqlite_backup_to_file($tmp_over); $over = undef; + $over_chg = 1; } if (!defined($new)) { # culled shard @@ -49,7 +52,6 @@ sub commit_changes ($$$$) { rename($old, "$new/old") or die "rename $old => $new/old: $!\n"; } - # Xtmpdir->DESTROY won't remove $new after this: rename($new, $old) or die "rename $new => $old: $!\n"; if (@st) { my $prev = "$old/old"; @@ -57,6 +59,10 @@ sub commit_changes ($$$$) { die "failed to remove $prev: $!\n"; } } + + # trigger ->check_inodes in read-only daemons + syswrite($im->{lockfh}, '.') if $over_chg; + remove_tree(@old_shard); $tmp = undef; if (!$opt->{-coarse_lock}) { @@ -75,7 +81,8 @@ sub commit_changes ($$$$) { $im->{shards} = $n; } } - + my $env = $opt->{-idx_env}; + local %ENV = (%ENV, %$env) if $env; PublicInbox::Admin::index_inbox($ibx, $im, $opt); } } @@ -85,7 +92,7 @@ sub cb_spawn { defined(my $pid = fork) or die "fork: $!"; return $pid if $pid > 0; $cb->($args, $opt); - exit 0; + POSIX::_exit(0); } sub runnable_or_die ($) { @@ -94,17 +101,17 @@ sub runnable_or_die ($) { } sub prepare_reindex ($$$) { - my ($ibx, $im, $reindex) = @_; - if ($ibx->{version} == 1) { + my ($ibx, $im, $opt) = @_; + if ($ibx->version == 1) { my $dir = $ibx->search->xdir(1); - my $xdb = Search::Xapian::Database->new($dir); + my $xdb = $PublicInbox::Search::X{Database}->new($dir); if (my $lc = $xdb->get_metadata('last_commit')) { - $reindex->{from} = $lc; + $opt->{reindex}->{from} = $lc; } } else { # v2 my $max; $im->git_dir_latest(\$max) or return; - my $from = $reindex->{from}; + my $from = $opt->{reindex}->{from}; my $mm = $ibx->mm; my $v = PublicInbox::Search::SCHEMA_VERSION(); foreach my $i (0..$max) { @@ -119,8 +126,14 @@ sub same_fs_or_die ($$) { die "$x and $y reside on different filesystems\n"; } +sub kill_pids { + my ($sig, $pids) = @_; + kill($sig, keys %$pids); # pids may be empty +} + sub process_queue { - my ($queue, $cb, $max, $opt) = @_; + my ($queue, $cb, $opt) = @_; + my $max = $opt->{jobs} // scalar(@$queue); if ($max <= 1) { while (defined(my $args = shift @$queue)) { $cb->($args, $opt); @@ -130,64 +143,58 @@ sub process_queue { # run in parallel: my %pids; + local %SIG = %SIG; + setup_signals(\&kill_pids, \%pids); while (@$queue) { while (scalar(keys(%pids)) < $max && scalar(@$queue)) { my $args = shift @$queue; $pids{cb_spawn($cb, $args, $opt)} = $args; } + my $flags = 0; while (scalar keys %pids) { - my $pid = waitpid(-1, 0); + my $pid = waitpid(-1, $flags) or last; + last if $pid < 0; my $args = delete $pids{$pid}; - die join(' ', @$args)." failed: $?\n" if $?; + if ($args) { + die join(' ', @$args)." failed: $?\n" if $?; + } else { + warn "unknown PID($pid) reaped: $?\n"; + } + $flags = WNOHANG if scalar(@$queue); } } } -sub run { - my ($ibx, $task, $opt) = @_; # task = 'cpdb' or 'compact' - my $cb = \&${\"PublicInbox::Xapcmd::$task"}; - PublicInbox::Admin::progress_prepare($opt ||= {}); - my $dir = $ibx->{inboxdir} or die "no inboxdir in inbox\n"; - runnable_or_die($XAPIAN_COMPACT) if $opt->{compact}; - my $reindex; # v1:{ from => $x40 }, v2:{ from => [ $x40, $x40, .. ] } } - my $from; # per-epoch ranges - - if (!$opt->{-coarse_lock}) { - $reindex = $opt->{reindex} = {}; - $from = $reindex->{from} = []; - require Search::Xapian::WritableDatabase; +sub prepare_run { + my ($ibx, $opt) = @_; + my $tmp = {}; # old shard dir => File::Temp->newdir object or undef + my @queue; # ([old//src,newdir]) - list of args for cpdb() or compact() + my $old; + if (my $srch = $ibx->search) { + $old = $srch->xdir(1); + -d $old or die "$old does not exist\n"; } - - $ibx->umask_prepare; - my $old = $ibx->search->xdir(1); - -d $old or die "$old does not exist\n"; - - my $tmp = PublicInbox::Xtmpdirs->new; - my $v = $ibx->{version} ||= 1; - my @q; my $reshard = $opt->{reshard}; if (defined $reshard && $reshard <= 0) { die "--reshard must be a positive number\n"; } - local %SIG = %SIG; - $tmp->setup_signals; - # we want temporary directories to be as deep as possible, # so v2 shards can keep "xap$SCHEMA_VERSION" on a separate FS. - if ($v == 1) { + if ($old && $ibx->version == 1) { if (defined $reshard) { warn "--reshard=$reshard ignored for v1 $ibx->{inboxdir}\n"; } - my $old_parent = dirname($old); - same_fs_or_die($old_parent, $old); + my $dir = dirname($old); + same_fs_or_die($dir, $old); my $v = PublicInbox::Search::SCHEMA_VERSION(); - my $wip = tempdir("xapian$v-XXXXXXXX", DIR => $old_parent); + my $wip = File::Temp->newdir("xapian$v-XXXXXXXX", DIR => $dir); $tmp->{$old} = $wip; - push @q, [ $old, $wip ]; - } else { + nodatacow_dir($wip->dirname); + push @queue, [ $old, $wip ]; + } elsif ($old) { opendir my $dh, $old or die "Failed to opendir $old: $!\n"; my @old_shards; while (defined(my $dn = readdir($dh))) { @@ -212,10 +219,11 @@ sub run { } foreach my $dn (0..$max_shard) { my $tmpl = "$dn-XXXXXXXX"; - my $wip = tempdir($tmpl, DIR => $old); - same_fs_or_die($old, $wip); + my $wip = File::Temp->newdir($tmpl, DIR => $old); + same_fs_or_die($old, $wip->dirname); my $cur = "$old/$dn"; - push @q, [ $src // $cur , $wip ]; + push @queue, [ $src // $cur , $wip ]; + nodatacow_dir($wip->dirname); $tmp->{$cur} = $wip; } # mark old shards to be unlinked @@ -223,27 +231,53 @@ sub run { $tmp->{$_} ||= undef for @$src; } } - my $max = $opt->{jobs} || scalar(@q); - $ibx->with_umask(sub { - my $im = $ibx->importer(0); - $im->lock_acquire; - - # fine-grained locking if we prepare for reindex - if (!$opt->{-coarse_lock}) { - prepare_reindex($ibx, $im, $reindex); - $im->lock_release; - } + ($tmp, \@queue); +} + +sub check_compact () { runnable_or_die($XAPIAN_COMPACT) } + +sub _run { + my ($ibx, $cb, $opt) = @_; + my $im = $ibx->importer(0); + $im->lock_acquire; + my ($tmp, $queue) = prepare_run($ibx, $opt); + + # fine-grained locking if we prepare for reindex + if (!$opt->{-coarse_lock}) { + prepare_reindex($ibx, $im, $opt); + $im->lock_release; + } - delete($ibx->{$_}) for (qw(mm over search)); # cleanup - process_queue(\@q, $cb, $max, $opt); - $im->lock_acquire if !$opt->{-coarse_lock}; - commit_changes($ibx, $im, $tmp, $opt); - }); + $ibx->cleanup; + process_queue($queue, $cb, $opt); + $im->lock_acquire if !$opt->{-coarse_lock}; + commit_changes($ibx, $im, $tmp, $opt); +} + +sub run { + my ($ibx, $task, $opt) = @_; # task = 'cpdb' or 'compact' + my $cb = \&$task; + PublicInbox::Admin::progress_prepare($opt ||= {}); + defined(my $dir = $ibx->{inboxdir}) or die "no inboxdir defined\n"; + -d $dir or die "inboxdir=$dir does not exist\n"; + check_compact() if $opt->{compact} && $ibx->search; + + if (!$opt->{-coarse_lock}) { + # per-epoch ranges for v2 + # v1:{ from => $OID }, v2:{ from => [ $OID, $OID, $OID ] } } + $opt->{reindex} = { from => $ibx->version == 1 ? '' : [] }; + PublicInbox::SearchIdx::load_xapian_writable(); + } + + local %SIG = %SIG; + setup_signals(); + $ibx->umask_prepare; + $ibx->with_umask(\&_run, $ibx, $cb, $opt); } sub cpdb_retryable ($$) { my ($src, $pfx) = @_; - if (ref($@) eq 'Search::Xapian::DatabaseModifiedError') { + if (ref($@) =~ /\bDatabaseModifiedError\b/) { warn "$pfx Xapian DB modified, reopening and retrying\n"; $src->reopen; return 1; @@ -263,11 +297,16 @@ sub progress_pfx ($) { ($p[-1] =~ /\A([0-9]+)/) ? "$p[-2]/$1" : $p[-1]; } +sub kill_compact { # setup_signals callback + my ($sig, $pidref) = @_; + kill($sig, $$pidref) if defined($$pidref); +} + # xapian-compact wrapper sub compact ($$) { my ($args, $opt) = @_; - my ($src, $dst) = @$args; - my ($r, $w); + my ($src, $newdir) = @$args; + my $dst = ref($newdir) ? $newdir->dirname : $newdir; my $pfx = $opt->{-progress_pfx} ||= progress_pfx($src); my $pr = $opt->{-progress}; my $rdr = {}; @@ -276,7 +315,6 @@ sub compact ($$) { defined(my $dfd = $opt->{$fd}) or next; $rdr->{$fd} = $dfd; } - $rdr->{1} = fileno($w) if $pr && pipe($r, $w); # we rely on --no-renumber to keep docids synched to NNTP my $cmd = [ $XAPIAN_COMPACT, '--no-renumber' ]; @@ -289,18 +327,18 @@ sub compact ($$) { } $pr->("$pfx `".join(' ', @$cmd)."'\n") if $pr; push @$cmd, $src, $dst; - my $pid = spawn($cmd, undef, $rdr); - if ($pr) { - close $w or die "close: \$w: $!"; - foreach (<$r>) { + my ($rd, $pid); + local %SIG = %SIG; + setup_signals(\&kill_compact, \$pid); + ($rd, $pid) = popen_rd($cmd, undef, $rdr); + while (<$rd>) { + if ($pr) { s/\r/\r$pfx /g; $pr->("$pfx $_"); } } - my $rp = waitpid($pid, 0); - if ($? || $rp != $pid) { - die join(' ', @$cmd)." failed: $? (pid=$pid, reaped=$rp)\n"; - } + waitpid($pid, 0); + die "@$cmd failed: \$?=$?\n" if $?; } sub cpdb_loop ($$$;$$) { @@ -348,9 +386,12 @@ sub cpdb_loop ($$$;$$) { # to the overhead of Perl. sub cpdb ($$) { my ($args, $opt) = @_; - my ($old, $new) = @$args; + my ($old, $newdir) = @$args; + my $new = $newdir->dirname; my ($src, $cur_shard); my $reshard; + PublicInbox::SearchIdx::load_xapian_writable() or die; + my $XapianDatabase = $PublicInbox::Search::X{Database}; if (ref($old) eq 'ARRAY') { ($cur_shard) = ($new =~ m!xap[0-9]+/([0-9]+)\b!); defined $cur_shard or @@ -361,33 +402,36 @@ sub cpdb ($$) { # resharding, M:N copy means have full read access foreach (@$old) { if ($src) { - my $sub = Search::Xapian::Database->new($_); + my $sub = $XapianDatabase->new($_); $src->add_database($sub); } else { - $src = Search::Xapian::Database->new($_); + $src = $XapianDatabase->new($_); } } } else { - $src = Search::Xapian::Database->new($old); + $src = $XapianDatabase->new($old); } - my ($xtmp, $tmp); + my ($tmp, $ft); local %SIG = %SIG; if ($opt->{compact}) { - my $newdir = dirname($new); - same_fs_or_die($newdir, $new); - $tmp = tempdir("$new.compact-XXXXXX", DIR => $newdir); - $xtmp = PublicInbox::Xtmpdirs->new; - $xtmp->setup_signals; - $xtmp->{$new} = $tmp; + my $dir = dirname($new); + same_fs_or_die($dir, $new); + $ft = File::Temp->newdir("$new.compact-XXXXXX", DIR => $dir); + setup_signals(); + $tmp = $ft->dirname; + nodatacow_dir($tmp); } else { $tmp = $new; } # like copydatabase(1), be sure we don't overwrite anything in case # of other bugs: - my $creat = Search::Xapian::DB_CREATE(); - my $dst = Search::Xapian::WritableDatabase->new($tmp, $creat); + my $flag = eval($PublicInbox::Search::Xap.'::DB_CREATE()'); + die if $@; + my $XapianWritableDatabase = $PublicInbox::Search::X{WritableDatabase}; + $flag |= $PublicInbox::SearchIdx::DB_NO_SYNC if !$opt->{fsync}; + my $dst = $XapianWritableDatabase->new($tmp, $flag); my $pr = $opt->{-progress}; my $pfx = $opt->{-progress_pfx} = progress_pfx($new); my $pr_data = { pr => $pr, pfx => $pfx, nr => 0 } if $pr; @@ -430,7 +474,7 @@ sub cpdb ($$) { # individually. $src = undef; foreach (@$old) { - my $old = Search::Xapian::Database->new($_); + my $old = $XapianDatabase->new($_); cpdb_loop($old, $dst, $pr_data, $cur_shard, $reshard); } } else { @@ -438,7 +482,7 @@ sub cpdb ($$) { } $pr->(sprintf($pr_data->{fmt}, $pr_data->{nr})) if $pr; - return unless $xtmp; + return unless $opt->{compact}; $src = $dst = undef; # flushes and closes @@ -446,33 +490,6 @@ sub cpdb ($$) { # since $dst isn't readable by HTTP or NNTP clients, yet: compact([ $tmp, $new ], $opt); remove_tree($tmp) or die "failed to remove $tmp: $!\n"; - $xtmp = undef; -} - -# slightly easier-to-manage manage than END{} blocks -package PublicInbox::Xtmpdirs; -use strict; -use warnings; -use File::Path qw(remove_tree); - -sub setup_signals () { - # http://www.tldp.org/LDP/abs/html/exitcodes.html - $SIG{INT} = sub { exit(130) }; - $SIG{HUP} = $SIG{PIPE} = $SIG{TERM} = sub { exit(1) }; -} - -sub new { - bless { '' => $$ }, $_[0]; # old shard => new (WIP) shard -} - -sub DESTROY { - my ($self) = @_; - my $owner_pid = delete($self->{''}) or return; - return if $owner_pid != $$; - foreach my $new (values %$self) { - defined $new or next; # may be undef if resharding - remove_tree($new) unless -d "$new/old"; - } } 1;