isearch: emulate per-inbox search with ->ALL

[public-inbox.git] / lib / PublicInbox / OverIdx.pm

diff --git a/lib/PublicInbox/OverIdx.pm b/lib/PublicInbox/OverIdx.pm
index dff2780d861f71326251744b383cebf9e10304ae..88daa64fd61c2b97274945d54c6e1d995edda5d8 100644 (file)
--- a/lib/PublicInbox/OverIdx.pm
+++ b/lib/PublicInbox/OverIdx.pm
@@ -170,8 +170,14 @@ sub _resolve_mid_to_tid {
                $$tid = $cur_tid;
        } else { # rethreading, queue up dead ghosts
                $$tid = next_tid($self);
-               my $num = $smsg->{num};
-               push(@{$self->{-ghosts_to_delete}}, $num) if $num < 0;
+               my $n = $smsg->{num};
+               if ($n > 0) {
+                       $self->{dbh}->prepare_cached(<<'')->execute($$tid, $n);
+UPDATE over SET tid = ? WHERE num = ?
+
+               } elsif ($n < 0) {
+                       push(@{$self->{-ghosts_to_delete}}, $n);
+               }
        }
        1;
 }
@@ -261,6 +267,13 @@ sub subject_path ($) {
        lc($subj);
 }
 
+sub ddd_for ($) {
+       my ($smsg) = @_;
+       my $dd = $smsg->to_doc_data;
+       utf8::encode($dd);
+       compress($dd);
+}
+
 sub add_overview {
        my ($self, $eml, $smsg) = @_;
        $smsg->{lines} = $eml->body_raw =~ tr!\n!\n!;
@@ -272,10 +285,7 @@ sub add_overview {
                $xpath = subject_path($subj);
                $xpath = id_compress($xpath);
        }
-       my $dd = $smsg->to_doc_data;
-       utf8::encode($dd);
-       $dd = compress($dd);
-       add_over($self, $smsg, $mids, $refs, $xpath, $dd);
+       add_over($self, $smsg, $mids, $refs, $xpath, ddd_for($smsg));
 }
 
 sub _add_over {
@@ -294,7 +304,7 @@ sub _add_over {
                }
        } elsif ($n < 0) { # ghost
                $$old_tid //= $cur_valid ? $cur_tid : next_tid($self);
-               link_refs($self, $refs, $$old_tid);
+               $$old_tid = link_refs($self, $refs, $$old_tid);
                delete_by_num($self, $n);
                $$v++;
        }
@@ -538,6 +548,11 @@ CREATE TABLE IF NOT EXISTS xref3 (
 
        $dbh->do('CREATE INDEX IF NOT EXISTS idx_docid ON xref3 (docid)');
 
+       # performance critical, this is not UNIQUE since we may need to
+       # tolerate some old bugs from indexing mirrors
+       $dbh->do('CREATE INDEX IF NOT EXISTS idx_nntp ON '.
+               'xref3 (oidbin,xnum,ibx_id)');
+
                $dbh->do(<<'');
 CREATE TABLE IF NOT EXISTS eidx_meta (
        key VARCHAR(255) PRIMARY KEY,
@@ -586,17 +601,60 @@ INSERT OR IGNORE INTO xref3 (docid, ibx_id, xnum, oidbin) VALUES (?, ?, ?, ?)
        $sth->execute;
 }
 
+# returns remaining reference count to $docid
 sub remove_xref3 {
-       my ($self, $docid, $oidhex, $eidx_key) = @_;
+       my ($self, $docid, $oidhex, $eidx_key, $rm_eidx_info) = @_;
        begin_lazy($self);
-       my $ibx_id = id_for($self, 'inboxes', 'ibx_id', eidx_key => $eidx_key);
        my $oidbin = pack('H*', $oidhex);
-       my $sth = $self->{dbh}->prepare_cached(<<'');
+       my ($sth, $ibx_id);
+       if (defined $eidx_key) {
+               $ibx_id = id_for($self, 'inboxes', 'ibx_id',
+                                       eidx_key => $eidx_key);
+               $sth = $self->{dbh}->prepare_cached(<<'');
 DELETE FROM xref3 WHERE docid = ? AND ibx_id = ? AND oidbin = ?
 
-       $sth->bind_param(1, $docid);
-       $sth->bind_param(2, $ibx_id);
-       $sth->bind_param(3, $oidbin, SQL_BLOB);
+               $sth->bind_param(1, $docid);
+               $sth->bind_param(2, $ibx_id);
+               $sth->bind_param(3, $oidbin, SQL_BLOB);
+       } else {
+               $sth = $self->{dbh}->prepare_cached(<<'');
+DELETE FROM xref3 WHERE docid = ? AND oidbin = ?
+
+               $sth->bind_param(1, $docid);
+               $sth->bind_param(2, $oidbin, SQL_BLOB);
+       }
+       $sth->execute;
+       $sth = $self->{dbh}->prepare_cached(<<'', undef, 1);
+SELECT COUNT(*) FROM xref3 WHERE docid = ?
+
+       $sth->execute($docid);
+       my $nr = $sth->fetchrow_array;
+       if ($nr == 0) {
+               delete_by_num($self, $docid);
+       } elsif (defined($ibx_id) && $rm_eidx_info) {
+               # if deduplication rules in ContentHash change, it's
+               # possible a docid can have multiple rows with the
+               # same ibx_id.  This governs whether or not we call
+               # ->shard_remove_eidx_info in ExtSearchIdx.
+               $sth = $self->{dbh}->prepare_cached(<<'', undef, 1);
+SELECT COUNT(*) FROM xref3 WHERE docid = ? AND ibx_id = ?
+
+               $sth->execute($docid, $ibx_id);
+               my $count = $sth->fetchrow_array;
+               $$rm_eidx_info = ($count == 0);
+       }
+       $nr;
+}
+
+# for when an xref3 goes missing, this does NOT update {ts}
+sub update_blob {
+       my ($self, $smsg, $oidhex) = @_;
+       my $sth = $self->{dbh}->prepare(<<'');
+UPDATE over SET ddd = ? WHERE num = ?
+
+       $smsg->{blob} = $oidhex;
+       $sth->bind_param(1, ddd_for($smsg), SQL_BLOB);
+       $sth->bind_param(2, $smsg->{num});
        $sth->execute;
 }