nntp: speed up mid_lookup() using ->ALL extindex

[public-inbox.git] / lib / PublicInbox / SearchIdxShard.pm

diff --git a/lib/PublicInbox/SearchIdxShard.pm b/lib/PublicInbox/SearchIdxShard.pm
index 644d8b58e65883c55ea09b4f780ba4736b2a9028..dcfeb0beec76c1c9e452cfad24a118e62c1cc007 100644 (file)
--- a/lib/PublicInbox/SearchIdxShard.pm
+++ b/lib/PublicInbox/SearchIdxShard.pm
@@ -10,6 +10,7 @@ use parent qw(PublicInbox::SearchIdx);
 use bytes qw(length);
 use IO::Handle (); # autoflush
 use PublicInbox::Eml;
+use PublicInbox::Sigfd;
 
 sub new {
        my ($class, $v2w, $shard) = @_; # v2w may be ExtSearchIdx
@@ -29,9 +30,13 @@ sub spawn_worker {
        my ($r, $w);
        pipe($r, $w) or die "pipe failed: $!\n";
        $w->autoflush(1);
+       my $oldset = PublicInbox::Sigfd::block_signals();
        my $pid = fork;
        defined $pid or die "fork failed: $!\n";
        if ($pid == 0) {
+               # these signals are localized in parent
+               $SIG{$_} = 'IGNORE' for (qw(TERM INT QUIT));
+               PublicInbox::Sigfd::sig_setmask($oldset);
                my $bnote = $v2w->atfork_child;
                close $w or die "failed to close: $!";
 
@@ -44,6 +49,7 @@ sub spawn_worker {
                die "unexpected MM $self->{mm}" if $self->{mm};
                exit;
        }
+       PublicInbox::Sigfd::sig_setmask($oldset);
        $self->{pid} = $pid;
        $self->{w} = $w;
        close $r or die "failed to close: $!";
@@ -77,18 +83,21 @@ sub shard_worker_loop ($$$$$) {
                } elsif ($line =~ s/\A\+X //) {
                        my ($len, $docid, $oid, $eidx_key) =
                                                        split(/ /, $line, 4);
+                       chomp $eidx_key;
                        $self->add_eidx_info($docid, $oid, $eidx_key,
                                                        eml($r, $len));
                } elsif ($line =~ s/\A-X //) {
                        my ($len, $docid, $oid, $eidx_key) =
                                                        split(/ /, $line, 4);
+                       chomp $eidx_key;
                        $self->remove_eidx_info($docid, $oid, $eidx_key,
                                                        eml($r, $len));
                } else {
                        chomp $line;
                        my $eidx_key;
-                       if ($line =~ s/\AX(.+)\0//) {
+                       if ($line =~ s/\AX=(.+)\0//) {
                                $eidx_key = $1;
+                               $v2w->{current_info} =~ s/\0/\\0 /;
                        }
                        # n.b. $mid may contain spaces(!)
                        my ($len, $bytes, $num, $oid, $ds, $ts, $tid, $mid)
@@ -113,14 +122,11 @@ sub shard_worker_loop ($$$$$) {
 sub index_raw {
        my ($self, $msgref, $eml, $smsg, $ibx) = @_;
        if (my $w = $self->{w}) {
-               if ($ibx) {
-                       print $w 'X', $ibx->eidx_key, "\0" or die
-                               "failed to write shard: $!\n";
-               }
+               my @ekey = $ibx ? ('X='.$ibx->eidx_key."\0") : ();
                $msgref //= \($eml->as_string);
                $smsg->{raw_bytes} //= length($$msgref);
                # mid must be last, it can contain spaces (but not LF)
-               print $w join(' ', @$smsg{qw(raw_bytes bytes
+               print $w @ekey, join(' ', @$smsg{qw(raw_bytes bytes
                                                num blob ds ts tid mid)}),
                        "\n", $$msgref or die "failed to write shard $!\n";
        } else {