extindex: fix delete (`d') handling

We need to completely remove a message from over.sqlite3 and
Xapian when no references remain, otherwise users will still see
the removed messages in NNTP overviews and WWW search
results/summaries.

References to messages are now solely handled by the `xref3'
table of over.sqlite3.  We can also trust `xref3' when deciding
whether to remove only the "O$eidx_key" and "G$lid" terms from a
document in Xapian or to remove the entire Xapian document.

diff --git a/lib/PublicInbox/ExtSearchIdx.pm b/lib/PublicInbox/ExtSearchIdx.pm
index cf90c562ebe27163248ded0ee0138f60d3132e03..d780776fbad5b66ba59e416d0a235c0c929f1e45 100644 (file)
--- a/lib/PublicInbox/ExtSearchIdx.pm
+++ b/lib/PublicInbox/ExtSearchIdx.pm
@@ -128,14 +128,21 @@ sub do_xpost ($$) {
        my $oid = $req->{oid};
        my $xibx = $req->{ibx};
        my $eml = $req->{eml};
+       my $eidx_key = $xibx->eidx_key;
        if (my $new_smsg = $req->{new_smsg}) { # 'm' on cross-posted message
                my $xnum = $req->{xnum};
-               $self->{oidx}->add_xref3($docid, $xnum, $oid, $xibx->eidx_key);
+               $self->{oidx}->add_xref3($docid, $xnum, $oid, $eidx_key);
                $idx->shard_add_eidx_info($docid, $oid, $xibx, $eml);
                check_batch_limit($req);
        } else { # 'd'
-               $self->{oidx}->remove_xref3($docid, $oid, $xibx->eidx_key);
-               $idx->shard_remove_eidx_info($docid, $oid, $xibx, $eml);
+               my $rm_eidx_info;
+               my $nr = $self->{oidx}->remove_xref3($docid, $oid, $eidx_key,
+                                                       \$rm_eidx_info);
+               if ($nr == 0) {
+                       $idx->shard_remove($oid, $docid);
+               } elsif ($rm_eidx_info) {
+                       $idx->shard_remove_eidx_info($docid, $oid, $xibx, $eml);
+               }
        }
 }
 

diff --git a/lib/PublicInbox/OverIdx.pm b/lib/PublicInbox/OverIdx.pm
index 8bec08da0b168746535bd32c445b87207fe72f5e..07cca4e5bb30d9c2053d9bb8ab62788571f2289b 100644 (file)
--- a/lib/PublicInbox/OverIdx.pm
+++ b/lib/PublicInbox/OverIdx.pm
@@ -595,13 +595,14 @@ INSERT OR IGNORE INTO xref3 (docid, ibx_id, xnum, oidbin) VALUES (?, ?, ?, ?)
        $sth->execute;
 }
 
+# returns remaining reference count to $docid
 sub remove_xref3 {
-       my ($self, $docid, $oidhex, $eidx_key) = @_;
+       my ($self, $docid, $oidhex, $eidx_key, $rm_eidx_info) = @_;
        begin_lazy($self);
        my $oidbin = pack('H*', $oidhex);
-       my $sth;
+       my ($sth, $ibx_id);
        if (defined $eidx_key) {
-               my $ibx_id = id_for($self, 'inboxes', 'ibx_id',
+               $ibx_id = id_for($self, 'inboxes', 'ibx_id',
                                        eidx_key => $eidx_key);
                $sth = $self->{dbh}->prepare_cached(<<'');
 DELETE FROM xref3 WHERE docid = ? AND ibx_id = ? AND oidbin = ?
@@ -617,6 +618,26 @@ DELETE FROM xref3 WHERE docid = ? AND oidbin = ?
                $sth->bind_param(2, $oidbin, SQL_BLOB);
        }
        $sth->execute;
+       $sth = $self->{dbh}->prepare_cached(<<'', undef, 1);
+SELECT COUNT(*) FROM xref3 WHERE docid = ?
+
+       $sth->execute($docid);
+       my $nr = $sth->fetchrow_array;
+       if ($nr == 0) {
+               delete_by_num($self, $docid);
+       } elsif (defined($ibx_id) && $rm_eidx_info) {
+               # if deduplication rules in ContentHash change, it's
+               # possible a docid can have multiple rows with the
+               # same ibx_id.  This governs whether or not we call
+               # ->shard_remove_eidx_info in ExtSearchIdx.
+               $sth = $self->{dbh}->prepare_cached(<<'', undef, 1);
+SELECT COUNT(*) FROM xref3 WHERE docid = ? AND ibx_id = ?
+
+               $sth->execute($docid, $ibx_id);
+               my $count = $sth->fetchrow_array;
+               $$rm_eidx_info = ($count == 0);
+       }
+       $nr;
 }
 
 # for when an xref3 goes missing, this does NOT update {ts}

diff --git a/t/extsearch.t b/t/extsearch.t
index f9f74e5c9db3ad6cf1cb06a477ff3dcb94a28bb3..f585555807f5cc4975115101a1848f010ce3868e 100644 (file)
--- a/t/extsearch.t
+++ b/t/extsearch.t
@@ -118,6 +118,26 @@ my $es = PublicInbox::ExtSearch->new("$home/extindex");
        is(scalar(@$x1), 1, 'original only has one xref3');
        is(scalar(@$x2), 1, 'new message has one xref3');
        isnt($x1->[0], $x2->[0], 'xref3 differs');
+
+       my $mset = $es->mset('b:"BEST MSG"');
+       is($mset->size, 1, 'new message found');
+       $mset = $es->mset('b:"test message"');
+       is($mset->size, 1, 'old message found');
+
+       delete @$es{qw(git over xdb)}; # fork preparation
+
+       open my $rmfh, '+>', undef or BAIL_OUT $!;
+       $rmfh->autoflush(1);
+       print $rmfh $eml2->as_string or BAIL_OUT $!;
+       seek($rmfh, 0, SEEK_SET) or BAIL_OUT $!;
+       $opt->{0} = $rmfh;
+       ok(run_script([qw(-learn rm --all)], undef, $opt), '-learn rm');
+
+       ok(run_script([qw(-extindex --all), "$home/extindex"], undef, undef),
+               'extindex after rm');
+       is($es->over->get_art(2), undef, 'doc #2 gone');
+       $mset = $es->mset('b:"BEST MSG"');
+       is($mset->size, 0, 'new message gone');
 }
 
 my $misc = $es->misc;

diff --git a/t/over.t b/t/over.t
index 56c20d01799ce7a244eaa82fc3abb13b524efebe..22061249e6370e1e79969a51add351416e275817 100644 (file)
--- a/t/over.t
+++ b/t/over.t
@@ -91,7 +91,7 @@ $over->eidx_prep;
                        'xref3 works forw two');
 
        @arg = qw(1349 adeadba7cafe example.key);
-       ok($over->remove_xref3(@arg), 'remove first');
+       is($over->remove_xref3(@arg), 1, 'remove first');
        $xref3 = $over->get_xref3(1349);
        is_deeply($xref3, [ 'example.kee:2018:deadbeefcafe' ],
                'confirm removal successful');