v2writable: more accurate {current_info} warnings/progress

[public-inbox.git] / lib / PublicInbox / ExtSearchIdx.pm

diff --git a/lib/PublicInbox/ExtSearchIdx.pm b/lib/PublicInbox/ExtSearchIdx.pm
index 050c4252d29f2e3f0c6da37041833b6694ab6f28..503428021854d827f5970a1c73510790d3c62976 100644 (file)
--- a/lib/PublicInbox/ExtSearchIdx.pm
+++ b/lib/PublicInbox/ExtSearchIdx.pm
@@ -30,6 +30,7 @@ use File::Spec;
 
 sub new {
        my (undef, $dir, $opt) = @_;
+       $dir = File::Spec->canonpath($dir);
        my $l = $opt->{indexlevel} // 'full';
        $l !~ $PublicInbox::SearchIdx::INDEXLEVELS and
                die "invalid indexlevel=$l\n";
@@ -106,6 +107,18 @@ sub is_bad_blob ($$$$) {
        $size == 0 ? 1 : 0; # size == 0 means purged
 }
 
+sub check_batch_limit ($) {
+       my ($req) = @_;
+       my $self = $req->{self};
+       my $new_smsg = $req->{new_smsg};
+
+       # {raw_bytes} may be unset, so just use {bytes}
+       my $n = $self->{transact_bytes} += $new_smsg->{bytes};
+
+       # set flag for PublicInbox::V2Writable::index_todo:
+       ${$req->{need_checkpoint}} = 1 if $n >= $self->{batch_bytes};
+}
+
 sub do_xpost ($$) {
        my ($req, $smsg) = @_;
        my $self = $req->{self};
@@ -118,6 +131,7 @@ sub do_xpost ($$) {
                my $xnum = $req->{xnum};
                $self->{oidx}->add_xref3($docid, $xnum, $oid, $xibx->eidx_key);
                $idx->shard_add_eidx_info($docid, $oid, $xibx, $eml);
+               check_batch_limit($req);
        } else { # 'd'
                $self->{oidx}->remove_xref3($docid, $oid, $xibx->eidx_key);
                $idx->shard_remove_eidx_info($docid, $oid, $xibx, $eml);
@@ -141,6 +155,7 @@ sub index_unseen ($) {
        my $ibx = delete $req->{ibx} or die 'BUG: {ibx} unset';
        $self->{oidx}->add_xref3($docid, $req->{xnum}, $oid, $ibx->eidx_key);
        $idx->index_raw(undef, $eml, $new_smsg, $ibx);
+       check_batch_limit($req);
 }
 
 sub do_finalize ($) {
@@ -150,7 +165,8 @@ sub do_finalize ($) {
        } elsif (exists $req->{new_smsg}) { # totally unseen messsage
                index_unseen($req);
        } else {
-               warn "W: ignoring delete $req->{oid} (not found)\n";
+               # `d' message was already unindexed in the v1/v2 inboxes,
+               # so it's too noisy to warn, here.
        }
 }
 
@@ -234,17 +250,22 @@ sub cur_ibx_xnum ($$) {
 
 sub index_oid { # git->cat_async callback for 'm'
        my ($bref, $oid, $type, $size, $req) = @_;
+       my $self = $req->{self};
+       local $self->{current_info} = "$self->{current_info} $oid";
        return if is_bad_blob($oid, $type, $size, $req->{oid});
        my $new_smsg = $req->{new_smsg} = bless {
                blob => $oid,
        }, 'PublicInbox::Smsg';
        $new_smsg->{bytes} = $size + crlf_adjust($$bref);
        defined($req->{xnum} = cur_ibx_xnum($req, $bref)) or return;
+       ++${$req->{nr}};
        do_step($req);
 }
 
 sub unindex_oid { # git->cat_async callback for 'd'
        my ($bref, $oid, $type, $size, $req) = @_;
+       my $self = $req->{self};
+       local $self->{current_info} = "$self->{current_info} $oid";
        return if is_bad_blob($oid, $type, $size, $req->{oid});
        return if defined(cur_ibx_xnum($req, $bref)); # was re-added
        do_step($req);
@@ -270,6 +291,8 @@ sub _sync_inbox ($$$) {
                -opt => $opt,
                self => $self,
                ibx => $ibx,
+               nr => \(my $nr = 0),
+               -regen_fmt => "%u/?\n",
        };
        my $v = $ibx->version;
        my $ekey = $ibx->eidx_key;
@@ -301,7 +324,11 @@ sub eidx_sync { # main entry point
        local $SIG{__WARN__} = sub {
                $warn_cb->($self->{current_info}, ': ', @_);
        };
-       _sync_inbox($self, $opt, $_) for (@{$self->{ibx_list}});
+
+       # don't use $_ here, it'll get clobbered by reindex_checkpoint
+       for my $ibx (@{$self->{ibx_list}}) {
+               _sync_inbox($self, $opt, $ibx);
+       }
 
        $self->{oidx}->rethread_done($opt);
 
@@ -362,8 +389,11 @@ sub idx_init { # similar to V2Writable
                $new{$line} = 1;
                push @new, $line;
        }
-       push @old, @new;
-       PublicInbox::V2Writable::write_alternates($info_dir, $mode, \@old);
+       if (scalar @new) {
+               push @old, @new;
+               my $o = \@old;
+               PublicInbox::V2Writable::write_alternates($info_dir, $mode, $o);
+       }
        $self->parallel_init($self->{indexlevel});
        $self->umask_prepare;
        $self->with_umask(\&PublicInbox::V2Writable::_idx_init, $self, $opt);