searchidxshard: special init for eidx

[public-inbox.git] / lib / PublicInbox / SearchIdx.pm

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 283bdd6c89782bb0c0d3b9926c8bd53cfcd9cda3..029b2726060a88229be332d51e3c3765b3f502b6 100644 (file)
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -368,9 +368,10 @@ sub add_xapian ($$$$) {
        $tg->set_document($doc);
        index_headers($self, $smsg);
 
-       if (my $ng_or_dir = $self->{ng_or_dir}) { # external index
+       if (defined(my $eidx_key = $smsg->{eidx_key})) {
+               $doc->add_boolean_term('O'.$eidx_key);
                $doc->add_boolean_term('P'.
-                               "$ng_or_dir:$smsg->{num}:$smsg->{blob}");
+                               "$eidx_key:$smsg->{num}:$smsg->{blob}");
        }
        msg_iter($eml, \&index_xapian, [ $self, $doc ]);
        index_ids($self, $doc, $eml, $mids);
@@ -456,43 +457,56 @@ sub _get_doc ($$$) {
 }
 
 sub add_xref3 {
-       my ($self, $docid, $xnum, $oid, $ng_or_dir, $eml) = @_;
+       my ($self, $docid, $xnum, $oid, $eidx_key, $eml) = @_;
        begin_txn_lazy($self);
        my $doc = _get_doc($self, $docid, $oid) or return;
        term_generator($self)->set_document($doc);
-       $doc->add_boolean_term('P'."$ng_or_dir:$xnum:$oid");
+       $doc->add_boolean_term('O'.$eidx_key);
+       $doc->add_boolean_term('P'."$eidx_key:$xnum:$oid");
        index_list_id($self, $doc, $eml);
        $self->{xdb}->replace_document($docid, $doc);
 }
 
 sub remove_xref3 {
-       my ($self, $docid, $oid, $ng_or_dir, $eml) = @_;
+       my ($self, $docid, $oid, $eidx_key, $eml) = @_;
        begin_txn_lazy($self);
        my $doc = _get_doc($self, $docid, $oid) or return;
        my $xref3 = PublicInbox::Smsg::xref3(undef, $doc);
-       for (grep(/\A\Q$ng_or_dir\E:[0-9]+:\Q$oid\E\z/, @$xref3)) {
+       my %x3 = map { $_ => undef } @$xref3;
+       for (grep(/\A\Q$eidx_key\E:[0-9]+:\Q$oid\E\z/, @$xref3)) {
+               delete $x3{$_};
                $doc->remove_term('P' . $_);
        }
-       for my $l ($eml->header_raw('List-Id')) {
-               $l =~ /<([^>]+)>/ or next;
-               my $lid = lc $1;
-               $doc->remove_term('G' . $lid);
-
-               # nb: we don't remove the XL probabilistic terms
-               # since terms may overlap if cross-posted.
-               #
-               # IOW, a message which has both <foo.example.com>
-               # and <bar.example.com> would have overlapping
-               # "XLexample" and "XLcom" as terms and which we
-               # wouldn't know if they're safe to remove if we just
-               # unindex <foo.example.com> while preserving
-               # <bar.example.com>.
-               #
-               # In any case, this entire sub is will likely never
-               # be needed and users using the "l:" prefix are probably
-               # rarer.
+       if (scalar(keys(%x3)) == 0) {
+               $self->{xdb}->delete_document($docid);
+               if (my $del_fh = $self->{del_fh}) { # TODO
+                       print $del_fh $docid, "\n" or die "E: print $!";
+               }
+       } else {
+               if (!grep(/\A\Q$eidx_key\E:/, keys %x3)) {
+                       $doc->remove_term('O'.$eidx_key);
+               }
+               for my $l ($eml->header_raw('List-Id')) {
+                       $l =~ /<([^>]+)>/ or next;
+                       my $lid = lc $1;
+                       $doc->remove_term('G' . $lid);
+
+                       # nb: we don't remove the XL probabilistic terms
+                       # since terms may overlap if cross-posted.
+                       #
+                       # IOW, a message which has both <foo.example.com>
+                       # and <bar.example.com> would have overlapping
+                       # "XLexample" and "XLcom" as terms and which we
+                       # wouldn't know if they're safe to remove if we just
+                       # unindex <foo.example.com> while preserving
+                       # <bar.example.com>.
+                       #
+                       # In any case, this entire sub is will likely never
+                       # be needed and users using the "l:" prefix are probably
+                       # rarer.
+               }
+               $self->{xdb}->replace_document($docid, $doc);
        }
-       $self->{xdb}->replace_document($docid, $doc);
 }
 
 sub get_val ($$) {
@@ -928,4 +942,17 @@ sub worker_done {
        die "$$ $0 still in transaction\n" if $self->{txn};
 }
 
+sub eidx_shard_new {
+       my ($class, $eidx, $shard) = @_;
+       my $self = bless {
+               xpfx => $eidx->{xpfx},
+               indexlevel => $eidx->{indexlevel},
+               -skip_docdata => 1,
+               shard => $shard,
+               creat => 1,
+       }, $class;
+       $self->{-set_indexlevel_once} = 1 if $self->{indexlevel} eq 'medium';
+       $self;
+}
+
 1;