extsearchidx: simplify reindex code paths

[public-inbox.git] / lib / PublicInbox / SearchIdx.pm

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 0124dd11b254f7df5abaaa490cad929e345ada6b..c6d2a0e88ba61dc9f7310aa40f2aab92966e621e 100644 (file)
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -15,7 +15,7 @@ use PublicInbox::InboxWritable;
 use PublicInbox::MID qw(mids_for_index mids);
 use PublicInbox::MsgIter;
 use PublicInbox::IdxStack;
-use Carp qw(croak);
+use Carp qw(croak carp);
 use POSIX qw(strftime);
 use Time::Local qw(timegm);
 use PublicInbox::OverIdx;
@@ -23,7 +23,7 @@ use PublicInbox::Spawn qw(spawn nodatacow_dir);
 use PublicInbox::Git qw(git_unquote);
 use PublicInbox::MsgTime qw(msg_timestamp msg_datestamp);
 our @EXPORT_OK = qw(crlf_adjust log2stack is_ancestor check_size prepare_stack
-       index_text term_generator add_val);
+       index_text term_generator add_val is_bad_blob);
 my $X = \%PublicInbox::Search::X;
 our ($DB_CREATE_OR_OPEN, $DB_OPEN);
 our $DB_NO_SYNC = 0;
@@ -352,8 +352,9 @@ sub index_ids ($$$$) {
        index_list_id($self, $doc, $hdr);
 }
 
-sub add_xapian ($$$$) {
+sub eml2doc ($$$;$) {
        my ($self, $eml, $smsg, $mids) = @_;
+       $mids //= mids_for_index($eml);
        my $doc = $X->{Document}->new;
        add_val($doc, PublicInbox::Search::TS(), $smsg->{ts});
        my @ds = gmtime($smsg->{ds});
@@ -396,6 +397,12 @@ sub add_xapian ($$$$) {
                        }
                }
        }
+       $doc;
+}
+
+sub add_xapian ($$$$) {
+       my ($self, $eml, $smsg, $mids) = @_;
+       my $doc = eml2doc($self, $eml, $smsg, $mids);
        $self->{xdb}->replace_document($smsg->{num}, $doc);
 }
 
@@ -591,8 +598,19 @@ sub crlf_adjust ($) {
        }
 }
 
+sub is_bad_blob ($$$$) {
+       my ($oid, $type, $size, $expect_oid) = @_;
+       if ($type ne 'blob') {
+               carp "W: $expect_oid is not a blob (type=$type)";
+               return 1;
+       }
+       croak "BUG: $oid != $expect_oid" if $oid ne $expect_oid;
+       $size == 0 ? 1 : 0; # size == 0 means purged
+}
+
 sub index_both { # git->cat_async callback
        my ($bref, $oid, $type, $size, $sync) = @_;
+       return if is_bad_blob($oid, $type, $size, $sync->{oid});
        my ($nr, $max) = @$sync{qw(nr max)};
        ++$$nr;
        $$max -= $size;
@@ -609,6 +627,7 @@ sub index_both { # git->cat_async callback
 
 sub unindex_both { # git->cat_async callback
        my ($bref, $oid, $type, $size, $sync) = @_;
+       return if is_bad_blob($oid, $type, $size, $sync->{oid});
        unindex_eml($sync->{sidx}, $oid, PublicInbox::Eml->new($bref));
        # may be undef if leftover
        if (defined(my $cur_cmt = $sync->{cur_cmt})) {
@@ -713,7 +732,7 @@ sub process_stack {
                $sync->{index_oid} = \&index_both;
        }
        while (my ($f, $at, $ct, $oid, $cur_cmt) = $stk->pop_rec) {
-               my $arg = { %$sync, cur_cmt => $cur_cmt };
+               my $arg = { %$sync, cur_cmt => $cur_cmt, oid => $oid };
                last if $sync->{quit};
                if ($f eq 'm') {
                        $arg->{autime} = $at;
@@ -929,6 +948,10 @@ sub set_metadata_once {
 
 sub _commit_txn {
        my ($self) = @_;
+       if (my $eidx = $self->{eidx}) {
+               $eidx->git->async_wait_all;
+               $eidx->{transact_bytes} = 0;
+       }
        if (my $xdb = $self->{xdb}) {
                set_metadata_once($self);
                $xdb->commit_transaction;