1 # Copyright (C) all contributors <meta@public-inbox.org>
2 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
4 # like PublicInbox::SearchIdx, but for searching for non-mail messages.
5 # Things indexed include:
8 # * (maybe) git code repository information
9 # Expect ~100K-1M documents with no parallelism opportunities,
10 # so no sharding, here.
12 # See MiscSearch for read-only counterpart
13 package PublicInbox::MiscIdx;
16 use PublicInbox::InboxWritable;
17 use PublicInbox::Search; # for SWIG Xapian and Search::Xapian compat
18 use PublicInbox::SearchIdx qw(index_text term_generator add_val);
21 use PublicInbox::MiscSearch;
22 use PublicInbox::Config;
23 use PublicInbox::Syscall;
27 my ($class, $eidx) = @_;
28 PublicInbox::SearchIdx::load_xapian_writable();
29 my $mi_dir = "$eidx->{xpfx}/misc";
30 File::Path::mkpath($mi_dir);
31 PublicInbox::Syscall::nodatacow_dir($mi_dir);
32 my $flags = $PublicInbox::SearchIdx::DB_CREATE_OR_OPEN;
33 $flags |= $PublicInbox::SearchIdx::DB_NO_SYNC if $eidx->{-no_fsync};
34 $flags |= $PublicInbox::SearchIdx::DB_DANGEROUS if $eidx->{-dangerous};
35 $json //= PublicInbox::Config::json();
39 indexlevel => 'full', # small DB, no point in medium?
45 my $wdb = $PublicInbox::Search::X{WritableDatabase};
46 my $xdb = eval { $wdb->new($self->{mi_dir}, $self->{flags}) };
47 croak "Failed opening $self->{mi_dir}: $@" if $@;
48 $xdb->begin_transaction;
54 my $xdb = delete $self->{xdb} or return;
55 $xdb->commit_transaction;
60 $self->{xdb} //= _begin_txn($self);
65 my ($self, $eidx_key) = @_;
66 my $xdb = $self->{xdb} //= _begin_txn($self);
67 my $head = $xdb->postlist_begin('Q'.$eidx_key);
68 my $tail = $xdb->postlist_end('Q'.$eidx_key);
69 my @docids; # only one, unless we had bugs
70 for (; $head != $tail; $head++) {
71 push @docids, $head->get_docid;
73 for my $docid (@docids) {
74 $xdb->delete_document($docid);
75 warn "I: remove inbox docid #$docid ($eidx_key)\n";
79 # adds or updates according to $eidx_key
81 my ($self, $ibx) = @_;
82 my $eidx_key = $ibx->eidx_key;
83 my $xdb = $self->{xdb} //= _begin_txn($self);
84 # Q = uniQue in Xapian terminology
85 my $head = $xdb->postlist_begin('Q'.$eidx_key);
86 my $tail = $xdb->postlist_end('Q'.$eidx_key);
88 for (; $head != $tail; $head++) {
90 my $i = $head->get_docid;
93 W: multiple inboxes keyed to `$eidx_key', deleting #$i
96 $docid = $head->get_docid;
99 $xdb->delete_document($_) for @drop; # just in case
101 my $doc = $PublicInbox::Search::X{Document}->new;
102 term_generator($self)->set_document($doc);
104 # allow sorting by modified and uidvalidity (created at)
105 add_val($doc, $PublicInbox::MiscSearch::MODIFIED, $ibx->modified);
106 add_val($doc, $PublicInbox::MiscSearch::UIDVALIDITY, $ibx->uidvalidity);
108 $doc->add_boolean_term('Q'.$eidx_key); # uniQue id
109 $doc->add_boolean_term('T'.'inbox'); # Type
111 if (defined($ibx->{newsgroup}) && $ibx->nntp_usable) {
112 $doc->add_boolean_term('T'.'newsgroup'); # additional Type
115 # force reread from disk, {description} could be loaded from {misc}
116 delete $ibx->{description};
117 my $desc = $ibx->description;
119 # description = S/Subject (or title)
121 index_text($self, $desc, 1, 'S');
122 index_text($self, $ibx->{name}, 1, 'XNAME');
126 infourl => 'XINFOURL',
129 while (my ($f, $pfx) = each %map) {
130 for my $v (@{$ibx->{$f} // []}) {
131 index_text($self, $v, 1, $pfx);
135 if (defined(my $max = $ibx->max_git_epoch)) { # v2
136 my $pfx = "/$ibx->{name}/git/";
137 for my $epoch (0..$max) {
138 my $git = $ibx->git_epoch($epoch) or return;
139 if (my $ent = $git->manifest_entry($epoch, $desc)) {
140 $data->{"$pfx$epoch.git"} = $ent;
141 $ent->{git_dir} = $git->{git_dir};
143 $git->cleanup; # ->modified starts cat-file --batch
145 } elsif (my $ent = $ibx->git->manifest_entry) { # v1
146 $ent->{git_dir} = $ibx->{inboxdir};
147 $data->{"/$ibx->{name}"} = $ent;
149 $doc->set_data($json->encode($data));
150 if (defined $docid) {
151 $xdb->replace_document($docid, $doc);
153 $xdb->add_document($doc);