extindex: fix w/ Xapian 1.2.21..1.2.24

[public-inbox.git] / lib / PublicInbox / SearchIdx.pm

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index d83fd4ca6b5abdf5d651a1b708f208990ea0b1a9..7f7b980d96a1efb665db1ebcc8f5c9935cdc0caa 100644 (file)
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -22,7 +22,7 @@ use PublicInbox::OverIdx;
 use PublicInbox::Spawn qw(spawn nodatacow_dir);
 use PublicInbox::Git qw(git_unquote);
 use PublicInbox::MsgTime qw(msg_timestamp msg_datestamp);
-our @EXPORT_OK = qw(crlf_adjust log2stack is_ancestor check_size prepare_stack
+our @EXPORT_OK = qw(log2stack is_ancestor check_size prepare_stack
        index_text term_generator add_val is_bad_blob);
 my $X = \%PublicInbox::Search::X;
 our ($DB_CREATE_OR_OPEN, $DB_OPEN);
@@ -105,8 +105,11 @@ sub load_xapian_writable () {
        $DB_CREATE_OR_OPEN = eval($xap.'::DB_CREATE_OR_OPEN()');
        $DB_OPEN = eval($xap.'::DB_OPEN()');
        my $ver = (eval($xap.'::major_version()') << 16) |
-               (eval($xap.'::minor_version()') << 8);
+               (eval($xap.'::minor_version()') << 8) |
+               eval($xap.'::revision()');
        $DB_NO_SYNC = 0x4 if $ver >= 0x10400;
+       # Xapian v1.2.21..v1.2.24 were missing close-on-exec on OFD locks
+       $X->{CLOEXEC_UNSET} = 1 if $ver >= 0x010215 && $ver <= 0x010218;
        1;
 }
 
@@ -397,6 +400,7 @@ sub eml2doc ($$$;$) {
 
 sub add_xapian ($$$$) {
        my ($self, $eml, $smsg, $mids) = @_;
+       begin_txn_lazy($self);
        my $doc = eml2doc($self, $eml, $smsg, $mids);
        $self->{xdb}->replace_document($smsg->{num}, $doc);
 }
@@ -552,6 +556,7 @@ sub smsg_from_doc ($) {
 
 sub xdb_remove {
        my ($self, @docids) = @_;
+       $self->begin_txn_lazy;
        my $xdb = $self->{xdb} or return;
        for my $docid (@docids) {
                eval { $xdb->delete_document($docid) };
@@ -559,13 +564,6 @@ sub xdb_remove {
        }
 }
 
-sub remove_by_docid {
-       my ($self, $num) = @_;
-       die "BUG: remove_by_docid is v2-only\n" if $self->{oidx};
-       $self->begin_txn_lazy;
-       xdb_remove($self, $num) if need_xapian($self);
-}
-
 sub index_git_blob_id {
        my ($doc, $pfx, $objid) = @_;
 
@@ -619,17 +617,6 @@ sub index_mm {
        }
 }
 
-# returns the number of bytes to add if given a non-CRLF arg
-sub crlf_adjust ($) {
-       if (index($_[0], "\r\n") < 0) {
-               # common case is LF-only, every \n needs an \r;
-               # so favor a cheap tr// over an expensive m//g
-               $_[0] =~ tr/\n/\n/;
-       } else { # count number of '\n' w/o '\r', expensive:
-               scalar(my @n = ($_[0] =~ m/(?<!\r)\n/g));
-       }
-}
-
 sub is_bad_blob ($$$$) {
        my ($oid, $type, $size, $expect_oid) = @_;
        if ($type ne 'blob') {
@@ -646,8 +633,8 @@ sub index_both { # git->cat_async callback
        my ($nr, $max) = @$sync{qw(nr max)};
        ++$$nr;
        $$max -= $size;
-       $size += crlf_adjust($$bref);
-       my $smsg = bless { bytes => $size, blob => $oid }, 'PublicInbox::Smsg';
+       my $smsg = bless { blob => $oid }, 'PublicInbox::Smsg';
+       $smsg->set_bytes($$bref, $size);
        my $self = $sync->{sidx};
        local $self->{current_info} = "$self->{current_info}: $oid";
        my $eml = PublicInbox::Eml->new($bref);