From d1915096bbd5fdda4818336b64131799966e9d6e Mon Sep 17 00:00:00 2001 From: "Eric Wong (Contractor, The Linux Foundation)" Date: Tue, 27 Feb 2018 00:41:21 +0000 Subject: [PATCH] rename SearchIdxThread to SearchIdxSkeleton Interchangably using "all", "skel", "threader", etc. were confusing. Standardize on the "skeleton" term to describe this class since it's also used for retrieval of basic headers. --- MANIFEST | 2 +- lib/PublicInbox/Search.pm | 2 +- lib/PublicInbox/SearchIdx.pm | 10 +++---- lib/PublicInbox/SearchIdxPart.pm | 4 +-- ...earchIdxThread.pm => SearchIdxSkeleton.pm} | 21 ++++++------- lib/PublicInbox/V2Writable.pm | 30 ++++++++++--------- 6 files changed, 36 insertions(+), 33 deletions(-) rename lib/PublicInbox/{SearchIdxThread.pm => SearchIdxSkeleton.pm} (85%) diff --git a/MANIFEST b/MANIFEST index 2a6f6f07..1aaf8fff 100644 --- a/MANIFEST +++ b/MANIFEST @@ -85,7 +85,7 @@ lib/PublicInbox/SaPlugin/ListMirror.pm lib/PublicInbox/Search.pm lib/PublicInbox/SearchIdx.pm lib/PublicInbox/SearchIdxPart.pm -lib/PublicInbox/SearchIdxThread.pm +lib/PublicInbox/SearchIdxSkeleton.pm lib/PublicInbox/SearchMsg.pm lib/PublicInbox/SearchThread.pm lib/PublicInbox/SearchView.pm diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm index 0f102dac..6b14942a 100644 --- a/lib/PublicInbox/Search.pm +++ b/lib/PublicInbox/Search.pm @@ -160,7 +160,7 @@ sub new { } warn "v2 repo with $parts found in $dir\n"; $self->{xdb} = $xdb; - $self->{skel} = Search::Xapian::Database->new("$dir/all"); + $self->{skel} = Search::Xapian::Database->new("$dir/skel"); } else { $self->{xdb} = Search::Xapian::Database->new($self->xdir); } diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm index b5d43d12..32594137 100644 --- a/lib/PublicInbox/SearchIdx.pm +++ b/lib/PublicInbox/SearchIdx.pm @@ -285,7 +285,7 @@ sub add_message { my ($doc_id, $old_tid); my $mid = mid_clean(mid_mime($mime)); - my $threader = $self->{threader}; + my $skel = $self->{skeleton}; eval { die 'Message-ID too long' if length($mid) > MAX_MID_SIZE; @@ -294,7 +294,7 @@ sub add_message { # convert a ghost to a regular message # it will also clobber any existing regular message $doc_id = $smsg->{doc_id}; - $old_tid = $smsg->thread_id unless $threader; + $old_tid = $smsg->thread_id unless $skel; } $smsg = PublicInbox::SearchMsg->new($mime); my $doc = $smsg->{doc}; @@ -362,9 +362,9 @@ sub add_message { # populates smsg->references for smsg->to_doc_data my $refs = parse_references($smsg); my $data = $smsg->to_doc_data($blob); - if ($threader) { + if ($skel) { push @values, $mid, $xpath, $data; - $threader->thread_msg(\@values); + $skel->index_skeleton(\@values); } else { link_message($self, $smsg, $refs, $old_tid); } @@ -817,7 +817,7 @@ sub DESTROY { $_[0]->{lockfh} = undef; } -# remote_* subs are only used by SearchIdxPart and SearchIdxThread: +# remote_* subs are only used by SearchIdxPart and SearchIdxSkeleton sub remote_commit { my ($self) = @_; print { $self->{w} } "commit\n" or die "failed to write commit: $!"; diff --git a/lib/PublicInbox/SearchIdxPart.pm b/lib/PublicInbox/SearchIdxPart.pm index 477a4f97..6025fc40 100644 --- a/lib/PublicInbox/SearchIdxPart.pm +++ b/lib/PublicInbox/SearchIdxPart.pm @@ -6,9 +6,9 @@ use warnings; use base qw(PublicInbox::SearchIdx); sub new { - my ($class, $v2writable, $part, $threader) = @_; + my ($class, $v2writable, $part, $skel) = @_; my $self = $class->SUPER::new($v2writable->{-inbox}, 1, $part); - $self->{threader} = $threader; + $self->{skeleton} = $skel; my ($r, $w); pipe($r, $w) or die "pipe failed: $!\n"; binmode $r, ':raw'; diff --git a/lib/PublicInbox/SearchIdxThread.pm b/lib/PublicInbox/SearchIdxSkeleton.pm similarity index 85% rename from lib/PublicInbox/SearchIdxThread.pm rename to lib/PublicInbox/SearchIdxSkeleton.pm index 6b50eb00..0016f89a 100644 --- a/lib/PublicInbox/SearchIdxThread.pm +++ b/lib/PublicInbox/SearchIdxSkeleton.pm @@ -1,6 +1,6 @@ # Copyright (C) 2018 all contributors # License: AGPL-3.0+ -package PublicInbox::SearchIdxThread; +package PublicInbox::SearchIdxSkeleton; use strict; use warnings; use base qw(PublicInbox::SearchIdx); @@ -8,7 +8,7 @@ use Storable qw(freeze thaw); sub new { my ($class, $v2writable) = @_; - my $self = $class->SUPER::new($v2writable->{-inbox}, 1, 'all'); + my $self = $class->SUPER::new($v2writable->{-inbox}, 1, 'skel'); # create the DB: $self->_xdb_acquire; $self->_xdb_release; @@ -23,8 +23,8 @@ sub new { $v2writable->atfork_child; $v2writable = undef; close $w; - eval { thread_worker_loop($self, $r) }; - die "thread worker died: $@\n" if $@; + eval { skeleton_worker_loop($self, $r) }; + die "skeleton worker died: $@\n" if $@; exit; } $self->{w} = $w; @@ -34,14 +34,14 @@ sub new { $w->autoflush(1); # lock on only exists in parent, not in worker - my $l = $self->{lock_path} = $self->xdir . '/thread.lock'; + my $l = $self->{lock_path} = $self->xdir . '/pi-v2-skeleton.lock'; open my $fh, '>>', $l or die "failed to create $l: $!\n"; $self; } -sub thread_worker_loop { +sub skeleton_worker_loop { my ($self, $r) = @_; - $0 = 'pi-v2-threader'; + $0 = 'pi-v2-skeleton'; my $msg; my $xdb = $self->_xdb_acquire; $xdb->begin_transaction; @@ -61,14 +61,14 @@ sub thread_worker_loop { $xdb->begin_transaction; $txn = 1; } - eval { $self->thread_msg_real($msg) }; + eval { index_skeleton_real($self, $msg) }; warn "failed to index message <$msg->[-1]>: $@\n" if $@; } } } # called by a partition worker -sub thread_msg { +sub index_skeleton { my ($self, $values) = @_; my $w = $self->{w}; my $err; @@ -83,7 +83,8 @@ sub thread_msg { die "print failed: $err\n" if $err; } -sub thread_msg_real { +# values: [ TS, NUM, BYTES, LINES, MID, XPATH, doc_data ] +sub index_skeleton_real ($$) { my ($self, $values) = @_; my $doc_data = pop @$values; my $xpath = pop @$values; diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm index 5e819da4..ff3b6573 100644 --- a/lib/PublicInbox/V2Writable.pm +++ b/lib/PublicInbox/V2Writable.pm @@ -7,7 +7,7 @@ use strict; use warnings; use Fcntl qw(:flock :DEFAULT); use PublicInbox::SearchIdxPart; -use PublicInbox::SearchIdxThread; +use PublicInbox::SearchIdxSkeleton; use PublicInbox::MIME; use PublicInbox::Git; use PublicInbox::Import; @@ -61,7 +61,7 @@ sub add { my ($len, $msgref) = @{$im->{last_object}}; $self->idx_init; - my $num = $self->{all}->index_mm($mime, 1); + my $num = $self->{skel}->index_mm($mime, 1); my $nparts = $self->{partitions}; my $part = $num % $nparts; my $idx = $self->idx_part($part); @@ -83,18 +83,18 @@ sub idx_init { my ($self) = @_; return if $self->{idx_parts}; - # first time initialization, first we create the threader pipe: - my $all = $self->{all} = PublicInbox::SearchIdxThread->new($self); + # first time initialization, first we create the skeleton pipe: + my $skel = $self->{skel} = PublicInbox::SearchIdxSkeleton->new($self); # need to create all parts before initializing msgmap FD my $max = $self->{partitions} - 1; my $idx = $self->{idx_parts} = []; for my $i (0..$max) { - push @$idx, PublicInbox::SearchIdxPart->new($self, $i, $all); + push @$idx, PublicInbox::SearchIdxPart->new($self, $i, $skel); } # Now that all subprocesses are up, we can open the FD for SQLite: - $all->_msgmap_init->{dbh}->begin_work; + $skel->_msgmap_init->{dbh}->begin_work; } sub remove { @@ -129,8 +129,8 @@ sub checkpoint { sub searchidx_checkpoint { my ($self, $more) = @_; - # order matters, we can only close {all} after all partitions - # are done because the partitions also write to {all} + # order matters, we can only close {skel} after all partitions + # are done because the partitions also write to {skel} if (my $parts = $self->{idx_parts}) { foreach my $idx (@$parts) { @@ -140,14 +140,16 @@ sub searchidx_checkpoint { delete $self->{idx_parts} unless $more; } - if (my $all = $self->{all}) { - $all->{mm}->{dbh}->commit; + if (my $skel = $self->{skel}) { + $skel->{mm}->{dbh}->commit; if ($more) { - $all->{mm}->{dbh}->begin_work; + $skel->{mm}->{dbh}->begin_work; + } + $skel->remote_commit; + unless ($more) { + $skel->remote_close; + delete $self->{skel}; } - $all->remote_commit; - $all->remote_close unless $more; - delete $self->{all} unless $more; } $self->{transact_bytes} = 0; } -- 2.44.0