lei: force --jobs=1,1 for SQLite < 3.8.3

[public-inbox.git] / lib / PublicInbox / OverIdx.pm

diff --git a/lib/PublicInbox/OverIdx.pm b/lib/PublicInbox/OverIdx.pm
index 0c8a4d9ee3f846d67cd514a41e1cbe8e916950be..a49ca6db3bb82d62a3e80c4738685cc68955d7b9 100644 (file)
--- a/lib/PublicInbox/OverIdx.pm
+++ b/lib/PublicInbox/OverIdx.pm
@@ -158,7 +158,8 @@ SELECT $cols FROM over WHERE over.num = ? LIMIT 1
 
                foreach (@$nums) {
                        $sth->execute($_->[0]);
-                       my $smsg = $sth->fetchrow_hashref;
+                       # $cb may delete rows and invalidate nums
+                       my $smsg = $sth->fetchrow_hashref // next;
                        $smsg = PublicInbox::Over::load_from_row($smsg);
                        $cb->($self, $smsg, @arg) or return;
                }
@@ -433,6 +434,7 @@ sub commit_lazy {
        my ($self) = @_;
        delete $self->{txn} or return;
        $self->{dbh}->commit;
+       eval { $self->{dbh}->do('PRAGMA optimize') };
 }
 
 sub begin_lazy {
@@ -542,9 +544,13 @@ CREATE TABLE IF NOT EXISTS xref3 (
        $dbh->do('CREATE INDEX IF NOT EXISTS idx_docid ON xref3 (docid)');
 
        # performance critical, this is not UNIQUE since we may need to
-       # tolerate some old bugs from indexing mirrors
-       $dbh->do('CREATE INDEX IF NOT EXISTS idx_nntp ON '.
-               'xref3 (oidbin,xnum,ibx_id)');
+       # tolerate some old bugs from indexing mirrors.  n.b. we used
+       # to index oidbin here, but leaving it out speeds up reindexing
+       # and "XHDR Xref <$MSGID>" isn't any slower w/o oidbin
+       $dbh->do('CREATE INDEX IF NOT EXISTS idx_reindex ON '.
+               'xref3 (xnum,ibx_id)');
+
+       $dbh->do('CREATE INDEX IF NOT EXISTS idx_oidbin ON xref3 (oidbin)');
 
                $dbh->do(<<'');
 CREATE TABLE IF NOT EXISTS eidx_meta (
@@ -601,50 +607,6 @@ INSERT OR IGNORE INTO xref3 (docid, ibx_id, xnum, oidbin) VALUES (?, ?, ?, ?)
        $sth->execute;
 }
 
-# returns remaining reference count to $docid
-sub remove_xref3 {
-       my ($self, $docid, $oidhex, $eidx_key, $rm_eidx_info) = @_;
-       begin_lazy($self);
-       my $oidbin = pack('H*', $oidhex);
-       my ($sth, $ibx_id);
-       if (defined $eidx_key) {
-               $ibx_id = ibx_id($self, $eidx_key);
-               $sth = $self->{dbh}->prepare_cached(<<'');
-DELETE FROM xref3 WHERE docid = ? AND ibx_id = ? AND oidbin = ?
-
-               $sth->bind_param(1, $docid);
-               $sth->bind_param(2, $ibx_id);
-               $sth->bind_param(3, $oidbin, SQL_BLOB);
-       } else {
-               $sth = $self->{dbh}->prepare_cached(<<'');
-DELETE FROM xref3 WHERE docid = ? AND oidbin = ?
-
-               $sth->bind_param(1, $docid);
-               $sth->bind_param(2, $oidbin, SQL_BLOB);
-       }
-       $sth->execute;
-       $sth = $self->{dbh}->prepare_cached(<<'', undef, 1);
-SELECT COUNT(*) FROM xref3 WHERE docid = ?
-
-       $sth->execute($docid);
-       my $nr = $sth->fetchrow_array;
-       if ($nr == 0) {
-               delete_by_num($self, $docid);
-       } elsif (defined($ibx_id) && $rm_eidx_info) {
-               # if deduplication rules in ContentHash change, it's
-               # possible a docid can have multiple rows with the
-               # same ibx_id.  This governs whether or not we call
-               # ->remove_eidx_info in ExtSearchIdx.
-               $sth = $self->{dbh}->prepare_cached(<<'', undef, 1);
-SELECT COUNT(*) FROM xref3 WHERE docid = ? AND ibx_id = ?
-
-               $sth->execute($docid, $ibx_id);
-               my $count = $sth->fetchrow_array;
-               $$rm_eidx_info = ($count == 0);
-       }
-       $nr;
-}
-
 # for when an xref3 goes missing, this does NOT update {ts}
 sub update_blob {
        my ($self, $smsg, $oidhex) = @_;
@@ -658,8 +620,7 @@ UPDATE over SET ddd = ? WHERE num = ?
 }
 
 sub merge_xref3 { # used for "-extindex --dedupe"
-       my ($self, $keep_docid, $drop_docid, $oidhex) = @_;
-       my $oidbin = pack('H*', $oidhex);
+       my ($self, $keep_docid, $drop_docid, $oidbin) = @_;
        my $sth = $self->{dbh}->prepare_cached(<<'');
 UPDATE OR IGNORE xref3 SET docid = ? WHERE docid = ? AND oidbin = ?
 
@@ -709,4 +670,16 @@ sub vivify_xvmd {
        $smsg->{-vivify_xvmd} = \@vivify_xvmd;
 }
 
+sub fork_ok {
+       return 1 if $DBD::SQLite::sqlite_version >= 3008003;
+       my ($opt) = @_;
+       my @j = split(/,/, $opt->{jobs} // '');
+       state $warned;
+       grep { $_ > 1 } @j and $warned //= warn('DBD::SQLite version is ',
+                $DBD::SQLite::sqlite_version,
+               ", need >= 3008003 (3.8.3) for --jobs > 1\n");
+       $opt->{jobs} = '1,1';
+       undef;
+}
+
 1;