miscidx: switch to lazy transactions

This fixes a sporadic failure on a 1/2 core VM where
"git cat-file --batch" hasn't started up by the time
$cleanup->() destroys the ALL.git directory in t/lei.t
(but not t/lei-oneshot.t).

This happens because dwaitpid() runs inside the event loop
asynchronously and we were able to return to the client before
the cat-file process could even start.

I could not reproduce this failure on my usual 4-core
workstation via "schedtool -a 0x1" to force the entire
test to use a single core.

Lazy transactions matches OverIdx and SearchIdx behavior, and
I've verified this lets us avoid problems with old Xapian
versions (on CentOS 7.x) which failed to set FD_CLOEXEC.

diff --git a/lib/PublicInbox/ExtSearchIdx.pm b/lib/PublicInbox/ExtSearchIdx.pm
index c782a62a83b6330eda77818e30dea8531d90190d..9b7340df6806dd2a9ca1179d2fb214a01d0c0ebf 100644 (file)
--- a/lib/PublicInbox/ExtSearchIdx.pm
+++ b/lib/PublicInbox/ExtSearchIdx.pm
@@ -1003,8 +1003,7 @@ sub idx_init { # similar to V2Writable
        $self->with_umask(\&_idx_init, $self, $opt);
        $self->{oidx}->begin_lazy;
        $self->{oidx}->eidx_prep;
-       $self->git->batch_prepare;
-       $self->{midx}->begin_txn;
+       $self->{midx}->create_xdb if @new;
 }
 
 sub _watch_commit { # PublicInbox::DS::add_timer callback

diff --git a/lib/PublicInbox/MiscIdx.pm b/lib/PublicInbox/MiscIdx.pm
index ab5e029a4625c6b77f16c9166de1d0e9afacbf3b..f5a374b2997a8c27ec9879f6f3c2695ebc2283b1 100644 (file)
--- a/lib/PublicInbox/MiscIdx.pm
+++ b/lib/PublicInbox/MiscIdx.pm
@@ -39,25 +39,30 @@ sub new {
        }, $class;
 }
 
-sub begin_txn {
+sub _begin_txn ($) {
        my ($self) = @_;
-       croak 'BUG: already in txn' if $self->{xdb}; # XXX make lazy?
        my $wdb = $PublicInbox::Search::X{WritableDatabase};
        my $xdb = eval { $wdb->new($self->{mi_dir}, $self->{flags}) };
        croak "Failed opening $self->{mi_dir}: $@" if $@;
-       $self->{xdb} = $xdb;
        $xdb->begin_transaction;
+       $xdb;
 }
 
 sub commit_txn {
        my ($self) = @_;
-       croak 'BUG: not in txn' unless $self->{xdb}; # XXX make lazy?
-       delete($self->{xdb})->commit_transaction;
+       my $xdb = delete $self->{xdb} or return;
+       $xdb->commit_transaction;
+}
+
+sub create_xdb {
+       my ($self) = @_;
+       $self->{xdb} //= _begin_txn($self);
+       commit_txn($self);
 }
 
 sub remove_eidx_key {
        my ($self, $eidx_key) = @_;
-       my $xdb = $self->{xdb};
+       my $xdb = $self->{xdb} //= _begin_txn($self);
        my $head = $xdb->postlist_begin('Q'.$eidx_key);
        my $tail = $xdb->postlist_end('Q'.$eidx_key);
        my @docids; # only one, unless we had bugs
@@ -74,7 +79,7 @@ sub remove_eidx_key {
 sub index_ibx {
        my ($self, $ibx) = @_;
        my $eidx_key = $ibx->eidx_key;
-       my $xdb = $self->{xdb};
+       my $xdb = $self->{xdb} //= _begin_txn($self);
        # Q = uniQue in Xapian terminology
        my $head = $xdb->postlist_begin('Q'.$eidx_key);
        my $tail = $xdb->postlist_end('Q'.$eidx_key);

diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index 0104f87ae421e25960220c5a64d9c5d30c41f136..7f9342b08d36941d1784615739f300ebf532542f 100644 (file)
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -623,10 +623,6 @@ shard[$i] bad echo:$echo != $i waiting for txn commit
                        $dbh->commit;
                        $dbh->begin_work;
                }
-               if ($midx) {
-                       $self->git->batch_prepare;
-                       $midx->begin_txn;
-               }
        }
        $self->{total_bytes} += $self->{transact_bytes};
        $self->{transact_bytes} = 0;

diff --git a/t/miscsearch.t b/t/miscsearch.t
index 0424328d5f9031a874459b84b3116020346025b4..413579fbc422b3ab5271bd67c14526159bf21d22 100644 (file)
--- a/t/miscsearch.t
+++ b/t/miscsearch.t
@@ -27,7 +27,6 @@ my $eidx = { xpfx => "$tmp/eidx", -no_fsync => 1 }; # mock ExtSearchIdx
        });
        $v1->init_inbox;
        my $mi = PublicInbox::MiscIdx->new($eidx);
-       $mi->begin_txn;
        $mi->index_ibx($v1);
        $mi->commit_txn;
 }