From: Eric Wong Date: Fri, 14 Jun 2019 17:35:04 +0000 (+0000) Subject: v2: rename SearchIdxPart => SearchIdxShard X-Git-Tag: v1.2.0~165^2~10 X-Git-Url: http://www.git.stargrave.org/?p=public-inbox.git;a=commitdiff_plain;h=0ffd4a9833da64006d558ef241badfef3c096d1b v2: rename SearchIdxPart => SearchIdxShard Another step towards keeping our file and package names consistent with Xapian terminology. --- diff --git a/MANIFEST b/MANIFEST index 3f0a79a6..c7693976 100644 --- a/MANIFEST +++ b/MANIFEST @@ -119,7 +119,7 @@ lib/PublicInbox/Reply.pm lib/PublicInbox/SaPlugin/ListMirror.pm lib/PublicInbox/Search.pm lib/PublicInbox/SearchIdx.pm -lib/PublicInbox/SearchIdxPart.pm +lib/PublicInbox/SearchIdxShard.pm lib/PublicInbox/SearchMsg.pm lib/PublicInbox/SearchThread.pm lib/PublicInbox/SearchView.pm diff --git a/lib/PublicInbox/SearchIdxPart.pm b/lib/PublicInbox/SearchIdxShard.pm similarity index 83% rename from lib/PublicInbox/SearchIdxPart.pm rename to lib/PublicInbox/SearchIdxShard.pm index 77fb7d90..15ec6578 100644 --- a/lib/PublicInbox/SearchIdxPart.pm +++ b/lib/PublicInbox/SearchIdxShard.pm @@ -3,23 +3,23 @@ # used to interface with a single Xapian shard in V2 repos. # See L for more info on how we shard Xapian -package PublicInbox::SearchIdxPart; +package PublicInbox::SearchIdxShard; use strict; use warnings; use base qw(PublicInbox::SearchIdx); sub new { - my ($class, $v2writable, $part) = @_; - my $self = $class->SUPER::new($v2writable->{-inbox}, 1, $part); + my ($class, $v2writable, $shard) = @_; + my $self = $class->SUPER::new($v2writable->{-inbox}, 1, $shard); # create the DB before forking: $self->_xdb_acquire; $self->_xdb_release; - $self->spawn_worker($v2writable, $part) if $v2writable->{parallel}; + $self->spawn_worker($v2writable, $shard) if $v2writable->{parallel}; $self; } sub spawn_worker { - my ($self, $v2writable, $part) = @_; + my ($self, $v2writable, $shard) = @_; my ($r, $w); pipe($r, $w) or die "pipe failed: $!\n"; binmode $r, ':raw'; @@ -35,8 +35,8 @@ sub spawn_worker { # speeds V2Writable batch imports across 8 cores by nearly 20% fcntl($r, 1031, 1048576) if $^O eq 'linux'; - eval { partition_worker_loop($self, $r, $part, $bnote) }; - die "worker $part died: $@\n" if $@; + eval { shard_worker_loop($self, $r, $shard, $bnote) }; + die "worker $shard died: $@\n" if $@; die "unexpected MM $self->{mm}" if $self->{mm}; exit; } @@ -45,14 +45,14 @@ sub spawn_worker { close $r or die "failed to close: $!"; } -sub partition_worker_loop ($$$$) { - my ($self, $r, $part, $bnote) = @_; - $0 = "pi-v2-shard[$part]"; +sub shard_worker_loop ($$$$) { + my ($self, $r, $shard, $bnote) = @_; + $0 = "pi-v2-shard[$shard]"; my $current_info = ''; my $warn_cb = $SIG{__WARN__} || sub { print STDERR @_ }; local $SIG{__WARN__} = sub { chomp $current_info; - $warn_cb->("[$part] $current_info: ", @_); + $warn_cb->("[$shard] $current_info: ", @_); }; $self->begin_txn_lazy; while (my $line = $r->getline) { @@ -64,7 +64,7 @@ sub partition_worker_loop ($$$$) { } elsif ($line eq "barrier\n") { $self->commit_txn_lazy; # no need to lock < 512 bytes is atomic under POSIX - print $bnote "barrier $part\n" or + print $bnote "barrier $shard\n" or die "write failed for barrier $!\n"; } elsif ($line =~ /\AD ([a-f0-9]{40,}) (.+)\n\z/s) { my ($oid, $mid) = ($1, $2); diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm index aa13aa8f..cc9ebfed 100644 --- a/lib/PublicInbox/V2Writable.pm +++ b/lib/PublicInbox/V2Writable.pm @@ -7,7 +7,7 @@ package PublicInbox::V2Writable; use strict; use warnings; use base qw(PublicInbox::Lock); -use PublicInbox::SearchIdxPart; +use PublicInbox::SearchIdxShard; use PublicInbox::MIME; use PublicInbox::Git; use PublicInbox::Import; @@ -300,7 +300,7 @@ sub idx_init { # idx_parts must be visible to all forked processes my $idx = $self->{idx_parts} = []; for my $i (0..$max) { - push @$idx, PublicInbox::SearchIdxPart->new($self, $i); + push @$idx, PublicInbox::SearchIdxShard->new($self, $i); } # Now that all subprocesses are up, we can open the FDs