with_umask: pass args to callback

[public-inbox.git] / lib / PublicInbox / SearchIdx.pm

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index a790ac4076acefbd6b50b68ae706fc2fe0ae40fc..c93c9034e9df31d557e516a0199e020264dcdd9c 100644 (file)
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -78,12 +78,12 @@ sub new {
 sub need_xapian ($) { $_[0]->{indexlevel} =~ $xapianlevels }
 
 sub _xdb_release {
-       my ($self) = @_;
+       my ($self, $wake) = @_;
        if (need_xapian($self)) {
                my $xdb = delete $self->{xdb} or croak 'not acquired';
                $xdb->close;
        }
-       $self->lock_release if $self->{creat};
+       $self->lock_release($wake) if $self->{creat};
        undef;
 }
 
@@ -549,11 +549,23 @@ sub unindex_mm {
        $self->{mm}->mid_delete(mid_mime($mime));
 }
 
+# returns the number of bytes to add if given a non-CRLF arg
+sub crlf_adjust ($) {
+       if (index($_[0], "\r\n") < 0) {
+               # common case is LF-only, every \n needs an \r;
+               # so favor a cheap tr// over an expensive m//g
+               $_[0] =~ tr/\n/\n/;
+       } else { # count number of '\n' w/o '\r', expensive:
+               scalar(my @n = ($_[0] =~ m/(?<!\r)\n/g));
+       }
+}
+
 sub index_both { # git->cat_async callback
        my ($bref, $oid, $type, $size, $sync) = @_;
        my ($nr, $max) = @$sync{qw(nr max)};
        ++$$nr;
        $$max -= $size;
+       $size += crlf_adjust($$bref);
        my $smsg = bless { bytes => $size, blob => $oid }, 'PublicInbox::Smsg';
        my $self = $sync->{sidx};
        my $eml = PublicInbox::Eml->new($bref);
@@ -573,7 +585,7 @@ sub unindex_both { # git->cat_async callback
 sub index_sync {
        my ($self, $opts) = @_;
        delete $self->{lock_path} if $opts->{-skip_lock};
-       $self->{-inbox}->with_umask(sub { $self->_index_sync($opts) })
+       $self->{-inbox}->with_umask(\&_index_sync, $self, $opts);
 }
 
 sub too_big ($$$) {
@@ -682,6 +694,7 @@ sub _git_log {
                } else {
                        # normal regen is for for fresh data
                        $self->{regen_down} = $fcount;
+                       $self->{regen_down} += $high unless $opts->{reindex};
                }
        } else {
                # Give oldest messages the smallest numbers
@@ -787,7 +800,7 @@ sub _index_sync {
                }
                $self->commit_txn_lazy;
                $git->cleanup;
-               $xdb = _xdb_release($self);
+               $xdb = _xdb_release($self, $nr);
                # let another process do some work... <
                $pr->("indexed $nr/$self->{ntodo}\n") if $pr && $nr;
                if (!$newest) {
@@ -841,17 +854,18 @@ sub remote_remove {
        }
 }
 
-sub begin_txn_lazy {
+sub _begin_txn {
        my ($self) = @_;
-       return if $self->{txn};
+       my $xdb = $self->{xdb} || $self->_xdb_acquire;
+       $self->{over}->begin_lazy if $self->{over};
+       $xdb->begin_transaction if $xdb;
+       $self->{txn} = 1;
+       $xdb;
+}
 
-       $self->{-inbox}->with_umask(sub {
-               my $xdb = $self->{xdb} || $self->_xdb_acquire;
-               $self->{over}->begin_lazy if $self->{over};
-               $xdb->begin_transaction if $xdb;
-               $self->{txn} = 1;
-               $xdb;
-       });
+sub begin_txn_lazy {
+       my ($self) = @_;
+       $self->{-inbox}->with_umask(\&_begin_txn, $self) if !$self->{txn};
 }
 
 # store 'indexlevel=medium' in v2 shard=0 and v1 (only one shard)
@@ -869,16 +883,19 @@ sub set_indexlevel {
        }
 }
 
+sub _commit_txn {
+       my ($self) = @_;
+       if (my $xdb = $self->{xdb}) {
+               set_indexlevel($self);
+               $xdb->commit_transaction;
+       }
+       $self->{over}->commit_lazy if $self->{over};
+}
+
 sub commit_txn_lazy {
        my ($self) = @_;
-       delete $self->{txn} or return;
-       $self->{-inbox}->with_umask(sub {
-               if (my $xdb = $self->{xdb}) {
-                       set_indexlevel($self);
-                       $xdb->commit_transaction;
-               }
-               $self->{over}->commit_lazy if $self->{over};
-       });
+       delete($self->{txn}) and
+               $self->{-inbox}->with_umask(\&_commit_txn, $self);
 }
 
 sub worker_done {