1 # Copyright (C) 2020 all contributors <meta@public-inbox.org>
2 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
4 # like PublicInbox::SearchIdx, but for searching for non-mail messages.
5 # Things indexed include:
8 # * (maybe) git code repository information
9 # Expect ~100K-1M documents with no parallelism opportunities,
10 # so no sharding, here.
12 # See MiscSearch for read-only counterpart
13 package PublicInbox::MiscIdx;
16 use PublicInbox::InboxWritable;
17 use PublicInbox::Search; # for SWIG Xapian and Search::Xapian compat
18 use PublicInbox::SearchIdx qw(index_text term_generator add_val);
19 use PublicInbox::Spawn qw(nodatacow_dir);
22 use PublicInbox::MiscSearch;
25 my ($class, $eidx) = @_;
26 PublicInbox::SearchIdx::load_xapian_writable();
27 my $mi_dir = "$eidx->{xpfx}/misc";
28 File::Path::mkpath($mi_dir);
29 nodatacow_dir($mi_dir);
30 my $flags = $PublicInbox::SearchIdx::DB_CREATE_OR_OPEN;
31 $flags |= $PublicInbox::SearchIdx::DB_NO_SYNC if $eidx->{-no_fsync};
35 indexlevel => 'full', # small DB, no point in medium?
41 croak 'BUG: already in txn' if $self->{xdb}; # XXX make lazy?
42 my $wdb = $PublicInbox::Search::X{WritableDatabase};
43 my $xdb = eval { $wdb->new($self->{mi_dir}, $self->{flags}) };
44 croak "Failed opening $self->{mi_dir}: $@" if $@;
46 $xdb->begin_transaction;
51 croak 'BUG: not in txn' unless $self->{xdb}; # XXX make lazy?
52 delete($self->{xdb})->commit_transaction;
56 my ($self, $ibx) = @_;
57 my $eidx_key = $ibx->eidx_key;
58 my $xdb = $self->{xdb};
59 # Q = uniQue in Xapian terminology
60 my $head = $xdb->postlist_begin('Q'.$eidx_key);
61 my $tail = $xdb->postlist_end('Q'.$eidx_key);
63 for (; $head != $tail; $head++) {
65 my $i = $head->get_docid;
68 W: multiple inboxes keyed to `$eidx_key', deleting #$i
71 $docid = $head->get_docid;
74 $xdb->delete_document($_) for @drop; # just in case
76 my $doc = $PublicInbox::Search::X{Document}->new;
78 # allow sorting by modified
79 add_val($doc, $PublicInbox::MiscSearch::MODIFIED, $ibx->modified);
81 $doc->add_boolean_term('Q'.$eidx_key);
82 $doc->add_boolean_term('T'.'inbox');
83 term_generator($self)->set_document($doc);
85 # description = S/Subject (or title)
87 index_text($self, $ibx->description, 1, 'S');
91 infourl => 'XINFOURL',
94 while (my ($f, $pfx) = each %map) {
95 for my $v (@{$ibx->{$f} // []}) {
96 index_text($self, $v, 1, $pfx);
99 index_text($self, $ibx->{name}, 1, 'XNAME');
100 if (defined $docid) {
101 $xdb->replace_document($docid, $doc);
103 $xdb->add_document($doc);