X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FXapcmd.pm;h=47a018d2b6e538ecc16f1338d43c1343ef6388a1;hb=fe7545b4682b8ffefc28c5b926fe601ecd83bd13;hp=19c6ff077d21911f003888edd2cc74a0b99f16e5;hpb=227a1d886672767e37cc86a3432952c14eb8a143;p=public-inbox.git diff --git a/lib/PublicInbox/Xapcmd.pm b/lib/PublicInbox/Xapcmd.pm index 19c6ff07..47a018d2 100644 --- a/lib/PublicInbox/Xapcmd.pm +++ b/lib/PublicInbox/Xapcmd.pm @@ -1,11 +1,11 @@ -# Copyright (C) 2018-2019 all contributors +# Copyright (C) 2018-2020 all contributors # License: AGPL-3.0+ package PublicInbox::Xapcmd; use strict; use warnings; use PublicInbox::Spawn qw(which popen_rd); use PublicInbox::Over; -use PublicInbox::SearchIdx; +use PublicInbox::SearchIdx qw(nodatacow_dir); use File::Temp 0.19 (); # ->newdir use File::Path qw(remove_tree); use File::Basename qw(dirname); @@ -23,6 +23,7 @@ sub commit_changes ($$$$) { $SIG{INT} or die 'BUG: $SIG{INT} not handled'; my @old_shard; + my $over_chg; while (my ($old, $newdir) = each %$tmp) { next if $old eq ''; # no invalid paths @@ -39,6 +40,7 @@ sub commit_changes ($$$$) { my $tmp_over = "$new/over.sqlite3"; $over->connect->sqlite_backup_to_file($tmp_over); $over = undef; + $over_chg = 1; } if (!defined($new)) { # culled shard @@ -59,6 +61,10 @@ sub commit_changes ($$$$) { die "failed to remove $prev: $!\n"; } } + + # trigger ->check_inodes in read-only daemons + syswrite($im->{lockfh}, '.') if $over_chg; + remove_tree(@old_shard); $tmp = undef; if (!$opt->{-coarse_lock}) { @@ -122,7 +128,8 @@ sub same_fs_or_die ($$) { } sub process_queue { - my ($queue, $cb, $max, $opt) = @_; + my ($queue, $cb, $opt) = @_; + my $max = $opt->{jobs} // scalar(@$queue); if ($max <= 1) { while (defined(my $args = shift @$queue)) { $cb->($args, $opt); @@ -141,7 +148,11 @@ sub process_queue { while (scalar keys %pids) { my $pid = waitpid(-1, 0); my $args = delete $pids{$pid}; - die join(' ', @$args)." failed: $?\n" if $?; + if ($args) { + die join(' ', @$args)." failed: $?\n" if $?; + } else { + warn "unknown PID($pid) reaped: $?\n"; + } } } } @@ -152,39 +163,23 @@ sub setup_signals () { $SIG{HUP} = $SIG{PIPE} = $SIG{TERM} = sub { exit(1) }; } -sub run { - my ($ibx, $task, $opt) = @_; # task = 'cpdb' or 'compact' - my $cb = \&${\"PublicInbox::Xapcmd::$task"}; - PublicInbox::Admin::progress_prepare($opt ||= {}); - my $dir = $ibx->{inboxdir} or die "no inboxdir in inbox\n"; - runnable_or_die($XAPIAN_COMPACT) if $opt->{compact}; - my $reindex; # v1:{ from => $x40 }, v2:{ from => [ $x40, $x40, .. ] } } - my $from; # per-epoch ranges - - if (!$opt->{-coarse_lock}) { - $reindex = $opt->{reindex} = {}; - $from = $reindex->{from} = []; - require PublicInbox::SearchIdx; - PublicInbox::SearchIdx::load_xapian_writable(); +sub prepare_run { + my ($ibx, $opt) = @_; + my $tmp = {}; # old shard dir => File::Temp->newdir object or undef + my @queue; # ([old//src,newdir]) - list of args for cpdb() or compact() + my $old; + if (my $srch = $ibx->search) { + $old = $srch->xdir(1); + -d $old or die "$old does not exist\n"; } - - $ibx->umask_prepare; - my $old = $ibx->search->xdir(1); - -d $old or die "$old does not exist\n"; - - my $tmp = {}; - my @q; my $reshard = $opt->{reshard}; if (defined $reshard && $reshard <= 0) { die "--reshard must be a positive number\n"; } - local %SIG = %SIG; - setup_signals(); - # we want temporary directories to be as deep as possible, # so v2 shards can keep "xap$SCHEMA_VERSION" on a separate FS. - if ($ibx->version == 1) { + if ($old && $ibx->version == 1) { if (defined $reshard) { warn "--reshard=$reshard ignored for v1 $ibx->{inboxdir}\n"; @@ -194,8 +189,9 @@ sub run { my $v = PublicInbox::Search::SCHEMA_VERSION(); my $wip = File::Temp->newdir("xapian$v-XXXXXXXX", DIR => $dir); $tmp->{$old} = $wip; - push @q, [ $old, $wip ]; - } else { + nodatacow_dir($wip->dirname); + push @queue, [ $old, $wip ]; + } elsif ($old) { opendir my $dh, $old or die "Failed to opendir $old: $!\n"; my @old_shards; while (defined(my $dn = readdir($dh))) { @@ -223,7 +219,8 @@ sub run { my $wip = File::Temp->newdir($tmpl, DIR => $old); same_fs_or_die($old, $wip->dirname); my $cur = "$old/$dn"; - push @q, [ $src // $cur , $wip ]; + push @queue, [ $src // $cur , $wip ]; + nodatacow_dir($wip->dirname); $tmp->{$cur} = $wip; } # mark old shards to be unlinked @@ -231,22 +228,49 @@ sub run { $tmp->{$_} ||= undef for @$src; } } - my $max = $opt->{jobs} || scalar(@q); - $ibx->with_umask(sub { - my $im = $ibx->importer(0); - $im->lock_acquire; - - # fine-grained locking if we prepare for reindex - if (!$opt->{-coarse_lock}) { - prepare_reindex($ibx, $im, $reindex); - $im->lock_release; - } + ($tmp, \@queue); +} - $ibx->cleanup; - process_queue(\@q, $cb, $max, $opt); - $im->lock_acquire if !$opt->{-coarse_lock}; - commit_changes($ibx, $im, $tmp, $opt); - }); +sub check_compact () { runnable_or_die($XAPIAN_COMPACT) } + +sub _run { + my ($ibx, $cb, $opt, $reindex) = @_; + my $im = $ibx->importer(0); + $im->lock_acquire; + my ($tmp, $queue) = prepare_run($ibx, $opt); + + # fine-grained locking if we prepare for reindex + if (!$opt->{-coarse_lock}) { + prepare_reindex($ibx, $im, $reindex); + $im->lock_release; + } + + $ibx->cleanup; + process_queue($queue, $cb, $opt); + $im->lock_acquire if !$opt->{-coarse_lock}; + commit_changes($ibx, $im, $tmp, $opt); +} + +sub run { + my ($ibx, $task, $opt) = @_; # task = 'cpdb' or 'compact' + my $cb = \&${\"PublicInbox::Xapcmd::$task"}; + PublicInbox::Admin::progress_prepare($opt ||= {}); + defined(my $dir = $ibx->{inboxdir}) or die "no inboxdir defined\n"; + -d $dir or die "inboxdir=$dir does not exist\n"; + check_compact() if $opt->{compact} && $ibx->search; + my $reindex; # v1:{ from => $x40 }, v2:{ from => [ $x40, $x40, .. ] } } + + if (!$opt->{-coarse_lock}) { + $reindex = $opt->{reindex} = { # per-epoch ranges for v2 + from => $ibx->version == 1 ? '' : [], + }; + PublicInbox::SearchIdx::load_xapian_writable(); + } + + local %SIG = %SIG; + setup_signals(); + $ibx->umask_prepare; + $ibx->with_umask(\&_run, $ibx, $cb, $opt, $reindex); } sub cpdb_retryable ($$) { @@ -385,16 +409,18 @@ sub cpdb ($$) { $ft = File::Temp->newdir("$new.compact-XXXXXX", DIR => $dir); setup_signals(); $tmp = $ft->dirname; + nodatacow_dir($tmp); } else { $tmp = $new; } # like copydatabase(1), be sure we don't overwrite anything in case # of other bugs: - my $creat = eval($PublicInbox::Search::Xap.'::DB_CREATE()'); + my $flag = eval($PublicInbox::Search::Xap.'::DB_CREATE()'); die if $@; my $XapianWritableDatabase = $PublicInbox::Search::X{WritableDatabase}; - my $dst = $XapianWritableDatabase->new($tmp, $creat); + $flag |= $PublicInbox::SearchIdx::DB_NO_SYNC if !$opt->{sync}; + my $dst = $XapianWritableDatabase->new($tmp, $flag); my $pr = $opt->{-progress}; my $pfx = $opt->{-progress_pfx} = progress_pfx($new); my $pr_data = { pr => $pr, pfx => $pfx, nr => 0 } if $pr;