index|extindex: support --dangerous flag

[public-inbox.git] / lib / PublicInbox / MiscIdx.pm

diff --git a/lib/PublicInbox/MiscIdx.pm b/lib/PublicInbox/MiscIdx.pm
index 64591d0516ddcfef135ee77c697f61b85012f583..5faf5c66ed1ebc850e99ded3bf320c899fc6e5f8 100644 (file)
--- a/lib/PublicInbox/MiscIdx.pm
+++ b/lib/PublicInbox/MiscIdx.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2020 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # like PublicInbox::SearchIdx, but for searching for non-mail messages.
@@ -16,20 +16,23 @@ use v5.10.1;
 use PublicInbox::InboxWritable;
 use PublicInbox::Search; # for SWIG Xapian and Search::Xapian compat
 use PublicInbox::SearchIdx qw(index_text term_generator add_val);
-use PublicInbox::Spawn qw(nodatacow_dir);
 use Carp qw(croak);
 use File::Path ();
 use PublicInbox::MiscSearch;
 use PublicInbox::Config;
+use PublicInbox::Syscall;
+my $json;
 
 sub new {
        my ($class, $eidx) = @_;
        PublicInbox::SearchIdx::load_xapian_writable();
        my $mi_dir = "$eidx->{xpfx}/misc";
        File::Path::mkpath($mi_dir);
-       nodatacow_dir($mi_dir);
+       PublicInbox::Syscall::nodatacow_dir($mi_dir);
        my $flags = $PublicInbox::SearchIdx::DB_CREATE_OR_OPEN;
        $flags |= $PublicInbox::SearchIdx::DB_NO_SYNC if $eidx->{-no_fsync};
+       $flags |= $PublicInbox::SearchIdx::DB_DANGEROUS if $eidx->{-dangerous};
+       $json //= PublicInbox::Config::json();
        bless {
                mi_dir => $mi_dir,
                flags => $flags,
@@ -37,25 +40,30 @@ sub new {
        }, $class;
 }
 
-sub begin_txn {
+sub _begin_txn ($) {
        my ($self) = @_;
-       croak 'BUG: already in txn' if $self->{xdb}; # XXX make lazy?
        my $wdb = $PublicInbox::Search::X{WritableDatabase};
        my $xdb = eval { $wdb->new($self->{mi_dir}, $self->{flags}) };
        croak "Failed opening $self->{mi_dir}: $@" if $@;
-       $self->{xdb} = $xdb;
        $xdb->begin_transaction;
+       $xdb;
 }
 
 sub commit_txn {
        my ($self) = @_;
-       croak 'BUG: not in txn' unless $self->{xdb}; # XXX make lazy?
-       delete($self->{xdb})->commit_transaction;
+       my $xdb = delete $self->{xdb} or return;
+       $xdb->commit_transaction;
+}
+
+sub create_xdb {
+       my ($self) = @_;
+       $self->{xdb} //= _begin_txn($self);
+       commit_txn($self);
 }
 
 sub remove_eidx_key {
        my ($self, $eidx_key) = @_;
-       my $xdb = $self->{xdb};
+       my $xdb = $self->{xdb} //= _begin_txn($self);
        my $head = $xdb->postlist_begin('Q'.$eidx_key);
        my $tail = $xdb->postlist_end('Q'.$eidx_key);
        my @docids; # only one, unless we had bugs
@@ -72,7 +80,7 @@ sub remove_eidx_key {
 sub index_ibx {
        my ($self, $ibx) = @_;
        my $eidx_key = $ibx->eidx_key;
-       my $xdb = $self->{xdb};
+       my $xdb = $self->{xdb} //= _begin_txn($self);
        # Q = uniQue in Xapian terminology
        my $head = $xdb->postlist_begin('Q'.$eidx_key);
        my $tail = $xdb->postlist_end('Q'.$eidx_key);
@@ -91,17 +99,27 @@ EOF
        $xdb->delete_document($_) for @drop; # just in case
 
        my $doc = $PublicInbox::Search::X{Document}->new;
+       term_generator($self)->set_document($doc);
 
-       # allow sorting by modified
+       # allow sorting by modified and uidvalidity (created at)
        add_val($doc, $PublicInbox::MiscSearch::MODIFIED, $ibx->modified);
+       add_val($doc, $PublicInbox::MiscSearch::UIDVALIDITY, $ibx->uidvalidity);
 
-       $doc->add_boolean_term('Q'.$eidx_key);
-       $doc->add_boolean_term('T'.'inbox');
-       term_generator($self)->set_document($doc);
+       $doc->add_boolean_term('Q'.$eidx_key); # uniQue id
+       $doc->add_boolean_term('T'.'inbox'); # Type
+
+       if (defined($ibx->{newsgroup}) && $ibx->nntp_usable) {
+               $doc->add_boolean_term('T'.'newsgroup'); # additional Type
+       }
+
+       # force reread from disk, {description} could be loaded from {misc}
+       delete $ibx->{description};
+       my $desc = $ibx->description;
 
        # description = S/Subject (or title)
        # address = A/Author
-       index_text($self, $ibx->description, 1, 'S');
+       index_text($self, $desc, 1, 'S');
+       index_text($self, $ibx->{name}, 1, 'XNAME');
        my %map = (
                address => 'A',
                listid => 'XLISTID',
@@ -113,10 +131,8 @@ EOF
                        index_text($self, $v, 1, $pfx);
                }
        }
-       index_text($self, $ibx->{name}, 1, 'XNAME');
        my $data = {};
        if (defined(my $max = $ibx->max_git_epoch)) { # v2
-               my $desc = $ibx->description;
                my $pfx = "/$ibx->{name}/git/";
                for my $epoch (0..$max) {
                        my $git = $ibx->git_epoch($epoch) or return;
@@ -130,7 +146,7 @@ EOF
                $ent->{git_dir} = $ibx->{inboxdir};
                $data->{"/$ibx->{name}"} = $ent;
        }
-       $doc->set_data(PublicInbox::Config::json()->encode($data));
+       $doc->set_data($json->encode($data));
        if (defined $docid) {
                $xdb->replace_document($docid, $doc);
        } else {