miscsearch: index UIDVALIDITY, use as startup cache

[public-inbox.git] / lib / PublicInbox / SearchIdx.pm

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 0fbe6560d793d463ac841976c25e642c607b0c4d..d1b0c724461271724f673d67e4bfa830a8034b08 100644 (file)
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -106,7 +106,6 @@ sub load_xapian_writable () {
        }
        eval 'require '.$X->{WritableDatabase} or die;
        *sortable_serialise = $xap.'::sortable_serialise';
-       *sortable_unserialise = $xap.'::sortable_unserialise';
        $DB_CREATE_OR_OPEN = eval($xap.'::DB_CREATE_OR_OPEN()');
        $DB_OPEN = eval($xap.'::DB_OPEN()');
        my $ver = (eval($xap.'::major_version()') << 16) |
@@ -352,8 +351,9 @@ sub index_ids ($$$$) {
        index_list_id($self, $doc, $hdr);
 }
 
-sub add_xapian ($$$$) {
+sub eml2doc ($$$;$) {
        my ($self, $eml, $smsg, $mids) = @_;
+       $mids //= mids_for_index($eml);
        my $doc = $X->{Document}->new;
        add_val($doc, PublicInbox::Search::TS(), $smsg->{ts});
        my @ds = gmtime($smsg->{ds});
@@ -396,6 +396,12 @@ sub add_xapian ($$$$) {
                        }
                }
        }
+       $doc;
+}
+
+sub add_xapian ($$$$) {
+       my ($self, $eml, $smsg, $mids) = @_;
+       my $doc = eml2doc($self, $eml, $smsg, $mids);
        $self->{xdb}->replace_document($smsg->{num}, $doc);
 }
 
@@ -494,17 +500,12 @@ sub remove_eidx_info {
        $self->{xdb}->replace_document($docid, $doc);
 }
 
-sub get_val ($$) {
-       my ($doc, $col) = @_;
-       sortable_unserialise($doc->get_value($col));
-}
-
 sub smsg_from_doc ($) {
        my ($doc) = @_;
        my $data = $doc->get_data or return;
        my $smsg = bless {}, 'PublicInbox::Smsg';
-       $smsg->{ts} = get_val($doc, PublicInbox::Search::TS());
-       my $dt = get_val($doc, PublicInbox::Search::DT());
+       $smsg->{ts} = int_val($doc, PublicInbox::Search::TS());
+       my $dt = int_val($doc, PublicInbox::Search::DT());
        my ($yyyy, $mon, $dd, $hh, $mm, $ss) = unpack('A4A2A2A2A2A2', $dt);
        $smsg->{ds} = timegm($ss, $mm, $hh, $dd, $mon - 1, $yyyy);
        $smsg->load_from_data($data);
@@ -610,6 +611,7 @@ sub index_both { # git->cat_async callback
        $size += crlf_adjust($$bref);
        my $smsg = bless { bytes => $size, blob => $oid }, 'PublicInbox::Smsg';
        my $self = $sync->{sidx};
+       local $self->{current_info} = "$self->{current_info}: $oid";
        my $eml = PublicInbox::Eml->new($bref);
        $smsg->{num} = index_mm($self, $eml, $oid, $sync) or
                die "E: could not generate NNTP article number for $oid";
@@ -621,7 +623,9 @@ sub index_both { # git->cat_async callback
 sub unindex_both { # git->cat_async callback
        my ($bref, $oid, $type, $size, $sync) = @_;
        return if is_bad_blob($oid, $type, $size, $sync->{oid});
-       unindex_eml($sync->{sidx}, $oid, PublicInbox::Eml->new($bref));
+       my $self = $sync->{sidx};
+       local $self->{current_info} = "$self->{current_info}: $oid";
+       unindex_eml($self, $oid, PublicInbox::Eml->new($bref));
        # may be undef if leftover
        if (defined(my $cur_cmt = $sync->{cur_cmt})) {
                ${$sync->{latest_cmt}} = $cur_cmt;
@@ -865,6 +869,7 @@ sub _index_sync {
        my ($self, $opt) = @_;
        my $tip = $opt->{ref} || 'HEAD';
        my $ibx = $self->{ibx};
+       local $self->{current_info} = "$ibx->{inboxdir}";
        $self->{batch_bytes} = $opt->{batch_size} // $BATCH_BYTES;
        $ibx->git->batch_prepare;
        my $pr = $opt->{-progress};
@@ -941,6 +946,10 @@ sub set_metadata_once {
 
 sub _commit_txn {
        my ($self) = @_;
+       if (my $eidx = $self->{eidx}) {
+               $eidx->git->async_wait_all;
+               $eidx->{transact_bytes} = 0;
+       }
        if (my $xdb = $self->{xdb}) {
                set_metadata_once($self);
                $xdb->commit_transaction;