X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FMiscIdx.pm;h=19200b92c910c8e22de43acccdbf1bd9e89fa0e6;hb=e28f33dc1b6e6a296946c0c58b43900c6cbd07f7;hp=acb49ce79acf041f43f76f050a07572d634e6c1b;hpb=2c012a3a2f95d9cf268bd92a61a53fc1633ba8f3;p=public-inbox.git
diff --git a/lib/PublicInbox/MiscIdx.pm b/lib/PublicInbox/MiscIdx.pm
index acb49ce7..19200b92 100644
--- a/lib/PublicInbox/MiscIdx.pm
+++ b/lib/PublicInbox/MiscIdx.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2020 all contributors
+# Copyright (C) all contributors
# License: AGPL-3.0+
# like PublicInbox::SearchIdx, but for searching for non-mail messages.
@@ -16,20 +16,23 @@ use v5.10.1;
use PublicInbox::InboxWritable;
use PublicInbox::Search; # for SWIG Xapian and Search::Xapian compat
use PublicInbox::SearchIdx qw(index_text term_generator add_val);
-use PublicInbox::Spawn qw(nodatacow_dir);
use Carp qw(croak);
use File::Path ();
use PublicInbox::MiscSearch;
use PublicInbox::Config;
+use PublicInbox::Syscall;
+my $json;
sub new {
my ($class, $eidx) = @_;
PublicInbox::SearchIdx::load_xapian_writable();
my $mi_dir = "$eidx->{xpfx}/misc";
File::Path::mkpath($mi_dir);
- nodatacow_dir($mi_dir);
+ PublicInbox::Syscall::nodatacow_dir($mi_dir);
my $flags = $PublicInbox::SearchIdx::DB_CREATE_OR_OPEN;
$flags |= $PublicInbox::SearchIdx::DB_NO_SYNC if $eidx->{-no_fsync};
+ $flags |= $PublicInbox::SearchIdx::DB_DANGEROUS if $eidx->{-dangerous};
+ $json //= PublicInbox::Config::json();
bless {
mi_dir => $mi_dir,
flags => $flags,
@@ -37,26 +40,47 @@ sub new {
}, $class;
}
-sub begin_txn {
+sub _begin_txn ($) {
my ($self) = @_;
- croak 'BUG: already in txn' if $self->{xdb}; # XXX make lazy?
my $wdb = $PublicInbox::Search::X{WritableDatabase};
my $xdb = eval { $wdb->new($self->{mi_dir}, $self->{flags}) };
croak "Failed opening $self->{mi_dir}: $@" if $@;
- $self->{xdb} = $xdb;
$xdb->begin_transaction;
+ $xdb;
}
sub commit_txn {
my ($self) = @_;
- croak 'BUG: not in txn' unless $self->{xdb}; # XXX make lazy?
- delete($self->{xdb})->commit_transaction;
+ my $xdb = delete $self->{xdb} or return;
+ $xdb->commit_transaction;
}
+sub create_xdb {
+ my ($self) = @_;
+ $self->{xdb} //= _begin_txn($self);
+ commit_txn($self);
+}
+
+sub remove_eidx_key {
+ my ($self, $eidx_key) = @_;
+ my $xdb = $self->{xdb} //= _begin_txn($self);
+ my $head = $xdb->postlist_begin('Q'.$eidx_key);
+ my $tail = $xdb->postlist_end('Q'.$eidx_key);
+ my @docids; # only one, unless we had bugs
+ for (; $head != $tail; $head++) {
+ push @docids, $head->get_docid;
+ }
+ for my $docid (@docids) {
+ $xdb->delete_document($docid);
+ warn "# remove inbox docid #$docid ($eidx_key)\n";
+ }
+}
+
+# adds or updates according to $eidx_key
sub index_ibx {
my ($self, $ibx) = @_;
my $eidx_key = $ibx->eidx_key;
- my $xdb = $self->{xdb};
+ my $xdb = $self->{xdb} //= _begin_txn($self);
# Q = uniQue in Xapian terminology
my $head = $xdb->postlist_begin('Q'.$eidx_key);
my $tail = $xdb->postlist_end('Q'.$eidx_key);
@@ -75,17 +99,31 @@ EOF
$xdb->delete_document($_) for @drop; # just in case
my $doc = $PublicInbox::Search::X{Document}->new;
+ term_generator($self)->set_document($doc);
- # allow sorting by modified
+ # allow sorting by modified and uidvalidity (created at)
add_val($doc, $PublicInbox::MiscSearch::MODIFIED, $ibx->modified);
+ add_val($doc, $PublicInbox::MiscSearch::UIDVALIDITY, $ibx->uidvalidity);
- $doc->add_boolean_term('Q'.$eidx_key);
- $doc->add_boolean_term('T'.'inbox');
- term_generator($self)->set_document($doc);
+ $doc->add_boolean_term('Q'.$eidx_key); # uniQue id
+ $doc->add_boolean_term('T'.'inbox'); # Type
+
+ # force reread from disk, {description} could be loaded from {misc}
+ delete @$ibx{qw(-art_min -art_max description)};
+ if (defined($ibx->{newsgroup}) && $ibx->nntp_usable) {
+ $doc->add_boolean_term('T'.'newsgroup'); # additional Type
+ my $n = $ibx->art_min;
+ add_val($doc, $PublicInbox::MiscSearch::ART_MIN, $n) if $n;
+ $n = $ibx->art_max;
+ add_val($doc, $PublicInbox::MiscSearch::ART_MAX, $n) if $n;
+ }
+
+ my $desc = $ibx->description;
# description = S/Subject (or title)
# address = A/Author
- index_text($self, $ibx->description, 1, 'S');
+ index_text($self, $desc, 1, 'S');
+ index_text($self, $ibx->{name}, 1, 'XNAME');
my %map = (
address => 'A',
listid => 'XLISTID',
@@ -97,22 +135,22 @@ EOF
index_text($self, $v, 1, $pfx);
}
}
- index_text($self, $ibx->{name}, 1, 'XNAME');
my $data = {};
if (defined(my $max = $ibx->max_git_epoch)) { # v2
- my $desc = $ibx->description;
my $pfx = "/$ibx->{name}/git/";
for my $epoch (0..$max) {
my $git = $ibx->git_epoch($epoch) or return;
if (my $ent = $git->manifest_entry($epoch, $desc)) {
$data->{"$pfx$epoch.git"} = $ent;
+ $ent->{git_dir} = $git->{git_dir};
}
$git->cleanup; # ->modified starts cat-file --batch
}
} elsif (my $ent = $ibx->git->manifest_entry) { # v1
+ $ent->{git_dir} = $ibx->{inboxdir};
$data->{"/$ibx->{name}"} = $ent;
}
- $doc->set_data(PublicInbox::Config::json()->encode($data));
+ $doc->set_data($json->encode($data));
if (defined $docid) {
$xdb->replace_document($docid, $doc);
} else {