nntp: art_lookup: use mid_lookup and simplify

[public-inbox.git] / lib / PublicInbox / OverIdx.pm

diff --git a/lib/PublicInbox/OverIdx.pm b/lib/PublicInbox/OverIdx.pm
index dff2780d861f71326251744b383cebf9e10304ae..07cca4e5bb30d9c2053d9bb8ab62788571f2289b 100644 (file)
--- a/lib/PublicInbox/OverIdx.pm
+++ b/lib/PublicInbox/OverIdx.pm
@@ -261,6 +261,13 @@ sub subject_path ($) {
        lc($subj);
 }
 
+sub ddd_for ($) {
+       my ($smsg) = @_;
+       my $dd = $smsg->to_doc_data;
+       utf8::encode($dd);
+       compress($dd);
+}
+
 sub add_overview {
        my ($self, $eml, $smsg) = @_;
        $smsg->{lines} = $eml->body_raw =~ tr!\n!\n!;
@@ -272,10 +279,7 @@ sub add_overview {
                $xpath = subject_path($subj);
                $xpath = id_compress($xpath);
        }
-       my $dd = $smsg->to_doc_data;
-       utf8::encode($dd);
-       $dd = compress($dd);
-       add_over($self, $smsg, $mids, $refs, $xpath, $dd);
+       add_over($self, $smsg, $mids, $refs, $xpath, ddd_for($smsg));
 }
 
 sub _add_over {
@@ -538,6 +542,11 @@ CREATE TABLE IF NOT EXISTS xref3 (
 
        $dbh->do('CREATE INDEX IF NOT EXISTS idx_docid ON xref3 (docid)');
 
+       # performance critical, this is not UNIQUE since we may need to
+       # tolerate some old bugs from indexing mirrors
+       $dbh->do('CREATE INDEX IF NOT EXISTS idx_nntp ON '.
+               'xref3 (oidbin,xnum,ibx_id)');
+
                $dbh->do(<<'');
 CREATE TABLE IF NOT EXISTS eidx_meta (
        key VARCHAR(255) PRIMARY KEY,
@@ -586,17 +595,60 @@ INSERT OR IGNORE INTO xref3 (docid, ibx_id, xnum, oidbin) VALUES (?, ?, ?, ?)
        $sth->execute;
 }
 
+# returns remaining reference count to $docid
 sub remove_xref3 {
-       my ($self, $docid, $oidhex, $eidx_key) = @_;
+       my ($self, $docid, $oidhex, $eidx_key, $rm_eidx_info) = @_;
        begin_lazy($self);
-       my $ibx_id = id_for($self, 'inboxes', 'ibx_id', eidx_key => $eidx_key);
        my $oidbin = pack('H*', $oidhex);
-       my $sth = $self->{dbh}->prepare_cached(<<'');
+       my ($sth, $ibx_id);
+       if (defined $eidx_key) {
+               $ibx_id = id_for($self, 'inboxes', 'ibx_id',
+                                       eidx_key => $eidx_key);
+               $sth = $self->{dbh}->prepare_cached(<<'');
 DELETE FROM xref3 WHERE docid = ? AND ibx_id = ? AND oidbin = ?
 
-       $sth->bind_param(1, $docid);
-       $sth->bind_param(2, $ibx_id);
-       $sth->bind_param(3, $oidbin, SQL_BLOB);
+               $sth->bind_param(1, $docid);
+               $sth->bind_param(2, $ibx_id);
+               $sth->bind_param(3, $oidbin, SQL_BLOB);
+       } else {
+               $sth = $self->{dbh}->prepare_cached(<<'');
+DELETE FROM xref3 WHERE docid = ? AND oidbin = ?
+
+               $sth->bind_param(1, $docid);
+               $sth->bind_param(2, $oidbin, SQL_BLOB);
+       }
+       $sth->execute;
+       $sth = $self->{dbh}->prepare_cached(<<'', undef, 1);
+SELECT COUNT(*) FROM xref3 WHERE docid = ?
+
+       $sth->execute($docid);
+       my $nr = $sth->fetchrow_array;
+       if ($nr == 0) {
+               delete_by_num($self, $docid);
+       } elsif (defined($ibx_id) && $rm_eidx_info) {
+               # if deduplication rules in ContentHash change, it's
+               # possible a docid can have multiple rows with the
+               # same ibx_id.  This governs whether or not we call
+               # ->shard_remove_eidx_info in ExtSearchIdx.
+               $sth = $self->{dbh}->prepare_cached(<<'', undef, 1);
+SELECT COUNT(*) FROM xref3 WHERE docid = ? AND ibx_id = ?
+
+               $sth->execute($docid, $ibx_id);
+               my $count = $sth->fetchrow_array;
+               $$rm_eidx_info = ($count == 0);
+       }
+       $nr;
+}
+
+# for when an xref3 goes missing, this does NOT update {ts}
+sub update_blob {
+       my ($self, $smsg, $oidhex) = @_;
+       my $sth = $self->{dbh}->prepare(<<'');
+UPDATE over SET ddd = ? WHERE num = ?
+
+       $smsg->{blob} = $oidhex;
+       $sth->bind_param(1, ddd_for($smsg), SQL_BLOB);
+       $sth->bind_param(2, $smsg->{num});
        $sth->execute;
 }