extindex: fix w/ Xapian 1.2.21..1.2.24

[public-inbox.git] / lib / PublicInbox / SearchIdx.pm

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 88349faa38b7a9325a11900155d4715c9faa90de..7f7b980d96a1efb665db1ebcc8f5c9935cdc0caa 100644 (file)
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -22,7 +22,7 @@ use PublicInbox::OverIdx;
 use PublicInbox::Spawn qw(spawn nodatacow_dir);
 use PublicInbox::Git qw(git_unquote);
 use PublicInbox::MsgTime qw(msg_timestamp msg_datestamp);
-our @EXPORT_OK = qw(crlf_adjust log2stack is_ancestor check_size prepare_stack
+our @EXPORT_OK = qw(log2stack is_ancestor check_size prepare_stack
        index_text term_generator add_val is_bad_blob);
 my $X = \%PublicInbox::Search::X;
 our ($DB_CREATE_OR_OPEN, $DB_OPEN);
@@ -105,8 +105,11 @@ sub load_xapian_writable () {
        $DB_CREATE_OR_OPEN = eval($xap.'::DB_CREATE_OR_OPEN()');
        $DB_OPEN = eval($xap.'::DB_OPEN()');
        my $ver = (eval($xap.'::major_version()') << 16) |
-               (eval($xap.'::minor_version()') << 8);
+               (eval($xap.'::minor_version()') << 8) |
+               eval($xap.'::revision()');
        $DB_NO_SYNC = 0x4 if $ver >= 0x10400;
+       # Xapian v1.2.21..v1.2.24 were missing close-on-exec on OFD locks
+       $X->{CLOEXEC_UNSET} = 1 if $ver >= 0x010215 && $ver <= 0x010218;
        1;
 }
 
@@ -397,6 +400,7 @@ sub eml2doc ($$$;$) {
 
 sub add_xapian ($$$$) {
        my ($self, $eml, $smsg, $mids) = @_;
+       begin_txn_lazy($self);
        my $doc = eml2doc($self, $eml, $smsg, $mids);
        $self->{xdb}->replace_document($smsg->{num}, $doc);
 }
@@ -414,6 +418,7 @@ sub _msgmap_init ($) {
 sub add_message {
        # mime = PublicInbox::Eml or Email::MIME object
        my ($self, $mime, $smsg, $sync) = @_;
+       begin_txn_lazy($self);
        my $mids = mids_for_index($mime);
        $smsg //= bless { blob => '' }, 'PublicInbox::Smsg'; # test-only compat
        $smsg->{mid} //= $mids->[0]; # v1 compatibility
@@ -551,6 +556,7 @@ sub smsg_from_doc ($) {
 
 sub xdb_remove {
        my ($self, @docids) = @_;
+       $self->begin_txn_lazy;
        my $xdb = $self->{xdb} or return;
        for my $docid (@docids) {
                eval { $xdb->delete_document($docid) };
@@ -558,13 +564,6 @@ sub xdb_remove {
        }
 }
 
-sub remove_by_docid {
-       my ($self, $num) = @_;
-       die "BUG: remove_by_docid is v2-only\n" if $self->{oidx};
-       $self->begin_txn_lazy;
-       xdb_remove($self, $num) if need_xapian($self);
-}
-
 sub index_git_blob_id {
        my ($doc, $pfx, $objid) = @_;
 
@@ -618,17 +617,6 @@ sub index_mm {
        }
 }
 
-# returns the number of bytes to add if given a non-CRLF arg
-sub crlf_adjust ($) {
-       if (index($_[0], "\r\n") < 0) {
-               # common case is LF-only, every \n needs an \r;
-               # so favor a cheap tr// over an expensive m//g
-               $_[0] =~ tr/\n/\n/;
-       } else { # count number of '\n' w/o '\r', expensive:
-               scalar(my @n = ($_[0] =~ m/(?<!\r)\n/g));
-       }
-}
-
 sub is_bad_blob ($$$$) {
        my ($oid, $type, $size, $expect_oid) = @_;
        if ($type ne 'blob') {
@@ -645,8 +633,8 @@ sub index_both { # git->cat_async callback
        my ($nr, $max) = @$sync{qw(nr max)};
        ++$$nr;
        $$max -= $size;
-       $size += crlf_adjust($$bref);
-       my $smsg = bless { bytes => $size, blob => $oid }, 'PublicInbox::Smsg';
+       my $smsg = bless { blob => $oid }, 'PublicInbox::Smsg';
+       $smsg->set_bytes($$bref, $size);
        my $self = $sync->{sidx};
        local $self->{current_info} = "$self->{current_info}: $oid";
        my $eml = PublicInbox::Eml->new($bref);
@@ -1002,14 +990,6 @@ sub commit_txn_lazy {
                $self->with_umask(\&_commit_txn, $self);
 }
 
-sub worker_done {
-       my ($self) = @_;
-       if (need_xapian($self)) {
-               die "$$ $0 xdb not released\n" if $self->{xdb};
-       }
-       die "$$ $0 still in transaction\n" if $self->{txn};
-}
-
 sub eidx_shard_new {
        my ($class, $eidx, $shard) = @_;
        my $self = bless {