support multiple CODE_URLs

[public-inbox.git] / lib / PublicInbox / SearchIdx.pm

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index cd8f4dd763b7bc20e1ee4ca0abe11ba1386e475d..cf2c2c55e381c1ca1f926198c8e8c9d13aae4161 100644 (file)
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -501,17 +501,18 @@ sub remove_eidx_info {
        $self->{xdb}->replace_document($docid, $doc);
 }
 
-sub get_val ($$) {
+sub int_val ($$) {
        my ($doc, $col) = @_;
-       sortable_unserialise($doc->get_value($col));
+       my $val = $doc->get_value($col) or return; # undefined is '' in Xapian
+       sortable_unserialise($val) + 0; # PV => IV conversion
 }
 
 sub smsg_from_doc ($) {
        my ($doc) = @_;
        my $data = $doc->get_data or return;
        my $smsg = bless {}, 'PublicInbox::Smsg';
-       $smsg->{ts} = get_val($doc, PublicInbox::Search::TS());
-       my $dt = get_val($doc, PublicInbox::Search::DT());
+       $smsg->{ts} = int_val($doc, PublicInbox::Search::TS());
+       my $dt = int_val($doc, PublicInbox::Search::DT());
        my ($yyyy, $mon, $dd, $hh, $mm, $ss) = unpack('A4A2A2A2A2A2', $dt);
        $smsg->{ds} = timegm($ss, $mm, $hh, $dd, $mon - 1, $yyyy);
        $smsg->load_from_data($data);
@@ -617,6 +618,7 @@ sub index_both { # git->cat_async callback
        $size += crlf_adjust($$bref);
        my $smsg = bless { bytes => $size, blob => $oid }, 'PublicInbox::Smsg';
        my $self = $sync->{sidx};
+       local $self->{current_info} = "$self->{current_info}: $oid";
        my $eml = PublicInbox::Eml->new($bref);
        $smsg->{num} = index_mm($self, $eml, $oid, $sync) or
                die "E: could not generate NNTP article number for $oid";
@@ -628,7 +630,9 @@ sub index_both { # git->cat_async callback
 sub unindex_both { # git->cat_async callback
        my ($bref, $oid, $type, $size, $sync) = @_;
        return if is_bad_blob($oid, $type, $size, $sync->{oid});
-       unindex_eml($sync->{sidx}, $oid, PublicInbox::Eml->new($bref));
+       my $self = $sync->{sidx};
+       local $self->{current_info} = "$self->{current_info}: $oid";
+       unindex_eml($self, $oid, PublicInbox::Eml->new($bref));
        # may be undef if leftover
        if (defined(my $cur_cmt = $sync->{cur_cmt})) {
                ${$sync->{latest_cmt}} = $cur_cmt;
@@ -872,6 +876,7 @@ sub _index_sync {
        my ($self, $opt) = @_;
        my $tip = $opt->{ref} || 'HEAD';
        my $ibx = $self->{ibx};
+       local $self->{current_info} = "$ibx->{inboxdir}";
        $self->{batch_bytes} = $opt->{batch_size} // $BATCH_BYTES;
        $ibx->git->batch_prepare;
        my $pr = $opt->{-progress};
@@ -1008,68 +1013,4 @@ SELECT COUNT(*) FROM over WHERE num = ?
        }
 }
 
-sub reindex_xap { # git->cat_async callback
-       my ($bref, $oid, $type, $size, $ary) = @_;
-       my ($ibx_id, $oidhex, $req, $more) = @$ary;
-       my $self = $req->{self} // die 'BUG: {self} missing';
-       my $eidx = $self->{eidx} // die 'BUG: {eidx} missing';
-       my $eidx_key = $self->{-eidx_key_for}->{$ibx_id} //
-                       die "BUG: bad ibx_id=$ibx_id ($oid)";
-
-       my $docid = $req->{docid};
-       local $eidx->{current_info} = "#$docid $oid";
-       return if is_bad_blob($oid, $type, $size, $oidhex);
-       if (my $doc = $req->{doc}) { # modify existing doc
-               $req->{tg_isset} //= do { # for existing documents in {xdb}
-                       term_generator($self)->set_document($doc);
-                       1;
-               };
-               $doc->add_boolean_term('O'.$eidx_key);
-               index_list_id($self, $doc, PublicInbox::Eml->new($bref));
-       } else { # first time seeing this doc
-               my $smsg = $self->{eidx}->over->get_art($docid) //
-                       die "BUG: #$docid ($oid) not in over";
-               $smsg->{bytes} = $size + crlf_adjust($$bref);
-               $smsg->{eidx_key} = $eidx_key;
-               my $eml = PublicInbox::Eml->new($bref);
-               $req->{doc} = eml2doc($self, $eml, $smsg);
-               $req->{tg_isset} = 1; # eml2doc calls $tg->set_document
-       }
-       return if $more;
-       my $doc = delete($req->{doc}) or return; # all bad blobs!
-       $eidx->{transact_bytes} += $size;
-       $self->{xdb}->replace_document($req->{docid}, $doc);
-}
-
-sub reindex_docid {
-       my ($self, $docid) = @_;
-       my $eidx = $self->{eidx} // die 'BUG: {eidx} missing';
-       my $eidx_key_for = $self->{-eidx_key_for} //= do {
-               my %eidx_key_for = map {
-                       $_->[0] => $_->[1];
-               } @{$eidx->over->dbh->selectall_arrayref(<<'')};
-SELECT ibx_id,eidx_key FROM inboxes
-
-               \%eidx_key_for;
-       };
-
-       begin_txn_lazy($self);
-       my $doc = eval { $self->{xdb}->get_document($docid) };
-       my $req = { doc => $doc, self => $self, docid => $docid };
-       my $sth = $eidx->over->dbh->prepare_cached(<<'', undef, 1);
-SELECT ibx_id,oidbin FROM xref3 WHERE docid = ? ORDER BY ibx_id ASC
-
-       $sth->execute($docid);
-       my $rows = $sth->fetchall_arrayref;
-       while (my $row = shift(@$rows)) {
-               my ($ibx_id, $oidbin) = @$row;
-               my $oidhex = unpack('H*', $oidbin);
-               $eidx->git->cat_async($oidhex, \&reindex_xap,
-                               [ $ibx_id, $oidhex, $req, scalar(@$rows) ]);
-       }
-       if ($eidx->{transact_bytes} >= $eidx->{batch_bytes}) {
-               commit_txn_lazy($self);
-       }
-}
-
 1;