1 # Copyright (C) 2018 all contributors <meta@public-inbox.org>
2 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
3 package PublicInbox::SearchIdxSkeleton;
6 use base qw(PublicInbox::SearchIdx);
7 use Storable qw(freeze thaw);
10 my ($class, $v2writable) = @_;
11 my $self = $class->SUPER::new($v2writable->{-inbox}, 1, 'skel');
17 pipe($r, $w) or die "pipe failed: $!\n";
21 defined $pid or die "fork failed: $!\n";
23 $v2writable->atfork_child;
26 eval { skeleton_worker_loop($self, $r) };
27 die "skeleton worker died: $@\n" if $@;
36 # lock on only exists in parent, not in worker
37 my $l = $self->{lock_path} = $self->xdir . '/pi-v2-skeleton.lock';
38 open my $fh, '>>', $l or die "failed to create $l: $!\n";
42 sub skeleton_worker_loop {
44 $0 = 'pi-v2-skeleton';
45 my $xdb = $self->_xdb_acquire;
46 $xdb->begin_transaction;
48 while (my $line = $r->getline) {
49 if ($line eq "commit\n") {
50 $xdb->commit_transaction if $txn;
52 } elsif ($line eq "close\n") {
57 my $n = read($r, my $msg, $len) or die "read: $!\n";
58 $n == $len or die "short read: $n != $len\n";
59 $msg = thaw($msg); # should raise on error
60 defined $msg or die "failed to thaw buffer\n";
61 $xdb ||= $self->_xdb_acquire;
63 $xdb->begin_transaction;
66 eval { index_skeleton_real($self, $msg) };
67 warn "failed to index message <$msg->[-1]>: $@\n" if $@;
70 die "xdb not released\n" if $xdb;
71 die "in transaction\n" if $txn;
74 # called by a partition worker
76 my ($self, $values) = @_;
79 my $str = freeze($values);
80 $str = length($str) . "\n" . $str;
82 # multiple processes write to the same pipe, so use flock
84 print $w $str or $err = $!;
87 die "print failed: $err\n" if $err;
90 # values: [ TS, NUM, BYTES, LINES, MID, XPATH, doc_data ]
91 sub index_skeleton_real ($$) {
92 my ($self, $values) = @_;
93 my $doc_data = pop @$values;
94 my $xpath = pop @$values;
95 my $mids = pop @$values;
96 my $ts = $values->[PublicInbox::Search::TS];
97 my $smsg = PublicInbox::SearchMsg->new(undef);
98 my $doc = $smsg->{doc};
99 $doc->add_term('XPATH' . $xpath) if defined $xpath;
100 foreach my $mid (@$mids) {
101 $doc->add_term('Q' . $mid);
103 PublicInbox::SearchIdx::add_values($doc, $values);
104 $doc->set_data($doc_data);
106 $smsg->load_from_data($doc_data);
107 my @refs = ($smsg->references =~ /<([^>]+)>/g);
108 $self->link_and_save($doc, $mids, \@refs);