nntp: xref: use ->ALL extindex if available

[public-inbox.git] / lib / PublicInbox / OverIdx.pm

diff --git a/lib/PublicInbox/OverIdx.pm b/lib/PublicInbox/OverIdx.pm
index 09bca790f20a719925e3437072bf1fbb3513f8e4..8bec08da0b168746535bd32c445b87207fe72f5e 100644 (file)
--- a/lib/PublicInbox/OverIdx.pm
+++ b/lib/PublicInbox/OverIdx.pm
@@ -261,6 +261,13 @@ sub subject_path ($) {
        lc($subj);
 }
 
+sub ddd_for ($) {
+       my ($smsg) = @_;
+       my $dd = $smsg->to_doc_data;
+       utf8::encode($dd);
+       compress($dd);
+}
+
 sub add_overview {
        my ($self, $eml, $smsg) = @_;
        $smsg->{lines} = $eml->body_raw =~ tr!\n!\n!;
@@ -272,10 +279,7 @@ sub add_overview {
                $xpath = subject_path($subj);
                $xpath = id_compress($xpath);
        }
-       my $dd = $smsg->to_doc_data;
-       utf8::encode($dd);
-       $dd = compress($dd);
-       add_over($self, $smsg, $mids, $refs, $xpath, $dd);
+       add_over($self, $smsg, $mids, $refs, $xpath, ddd_for($smsg));
 }
 
 sub _add_over {
@@ -517,20 +521,32 @@ sub eidx_prep ($) {
        my ($self) = @_;
        $self->{-eidx_prep} //= do {
                my $dbh = $self->dbh;
-               $dbh->do(<<'');
-INSERT OR IGNORE INTO counter (key) VALUES ('oidmap_num')
+               $dbh->do(<<"");
+INSERT OR IGNORE INTO counter (key) VALUES ('eidx_docid')
 
                $dbh->do(<<'');
-INSERT OR IGNORE INTO counter (key) VALUES ('eidx_docid')
+CREATE TABLE IF NOT EXISTS inboxes (
+       ibx_id INTEGER PRIMARY KEY AUTOINCREMENT,
+       eidx_key VARCHAR(255) NOT NULL, /* {newsgroup} // {inboxdir} */
+       UNIQUE (eidx_key)
+)
 
                $dbh->do(<<'');
-CREATE TABLE IF NOT EXISTS oidmap (
-       num INTEGER NOT NULL, /* NNTP article number == IMAP UID */
-       oidbin VARBINARY, /* 20-byte SHA-1 or 32-byte SHA-256 */
-       UNIQUE (num),
-       UNIQUE (oidbin)
+CREATE TABLE IF NOT EXISTS xref3 (
+       docid INTEGER NOT NULL, /* <=> over.num */
+       ibx_id INTEGER NOT NULL, /* <=> inboxes.ibx_id */
+       xnum INTEGER NOT NULL, /* NNTP article number in ibx */
+       oidbin VARBINARY NOT NULL, /* 20-byte SHA-1 or 32-byte SHA-256 */
+       UNIQUE (docid, ibx_id, xnum, oidbin)
 )
 
+       $dbh->do('CREATE INDEX IF NOT EXISTS idx_docid ON xref3 (docid)');
+
+       # performance critical, this is not UNIQUE since we may need to
+       # tolerate some old bugs from indexing mirrors
+       $dbh->do('CREATE INDEX IF NOT EXISTS idx_nntp ON '.
+               'xref3 (oidbin,xnum,ibx_id)');
+
                $dbh->do(<<'');
 CREATE TABLE IF NOT EXISTS eidx_meta (
        key VARCHAR(255) PRIMARY KEY,
@@ -564,28 +580,55 @@ sub eidx_max {
        get_counter($self->{dbh}, 'eidx_docid');
 }
 
-sub oid2num {
-       my ($self, $oidhex) = @_;
-       my $dbh = eidx_prep($self);
-       my $sth = $dbh->prepare_cached(<<'', undef, 1);
-SELECT num FROM oidmap WHERE oidbin = ?
+sub add_xref3 {
+       my ($self, $docid, $xnum, $oidhex, $eidx_key) = @_;
+       begin_lazy($self);
+       my $ibx_id = id_for($self, 'inboxes', 'ibx_id', eidx_key => $eidx_key);
+       my $oidbin = pack('H*', $oidhex);
+       my $sth = $self->{dbh}->prepare_cached(<<'');
+INSERT OR IGNORE INTO xref3 (docid, ibx_id, xnum, oidbin) VALUES (?, ?, ?, ?)
+
+       $sth->bind_param(1, $docid);
+       $sth->bind_param(2, $ibx_id);
+       $sth->bind_param(3, $xnum);
+       $sth->bind_param(4, $oidbin, SQL_BLOB);
+       $sth->execute;
+}
 
-       $sth->bind_param(1, pack('H*', $oidhex), SQL_BLOB);
+sub remove_xref3 {
+       my ($self, $docid, $oidhex, $eidx_key) = @_;
+       begin_lazy($self);
+       my $oidbin = pack('H*', $oidhex);
+       my $sth;
+       if (defined $eidx_key) {
+               my $ibx_id = id_for($self, 'inboxes', 'ibx_id',
+                                       eidx_key => $eidx_key);
+               $sth = $self->{dbh}->prepare_cached(<<'');
+DELETE FROM xref3 WHERE docid = ? AND ibx_id = ? AND oidbin = ?
+
+               $sth->bind_param(1, $docid);
+               $sth->bind_param(2, $ibx_id);
+               $sth->bind_param(3, $oidbin, SQL_BLOB);
+       } else {
+               $sth = $self->{dbh}->prepare_cached(<<'');
+DELETE FROM xref3 WHERE docid = ? AND oidbin = ?
+
+               $sth->bind_param(1, $docid);
+               $sth->bind_param(2, $oidbin, SQL_BLOB);
+       }
        $sth->execute;
-       $sth->fetchrow_array;
 }
 
-sub oid_add {
-       my ($self, $oidhex) = @_;
-       my $dbh = eidx_prep($self);
-       my $num = adj_counter($self, 'oidmap_num', '+');
-       my $sth = $dbh->prepare_cached(<<'');
-INSERT INTO oidmap (num, oidbin) VALUES (?,?)
+# for when an xref3 goes missing, this does NOT update {ts}
+sub update_blob {
+       my ($self, $smsg, $oidhex) = @_;
+       my $sth = $self->{dbh}->prepare(<<'');
+UPDATE over SET ddd = ? WHERE num = ?
 
-       $sth->bind_param(1, $num);
-       $sth->bind_param(2, pack('H*', $oidhex), SQL_BLOB);
+       $smsg->{blob} = $oidhex;
+       $sth->bind_param(1, ddd_for($smsg), SQL_BLOB);
+       $sth->bind_param(2, $smsg->{num});
        $sth->execute;
-       $num;
 }
 
 1;