lib/PublicInbox/Config.pm | 3 +++ lib/PublicInbox/Inbox.pm | 2 ++ lib/PublicInbox/InboxWritable.pm | 2 -- lib/PublicInbox/ManifestJsGz.pm | 39 ++++++++++++++++++++++++++++++++------- lib/PublicInbox/MiscSearch.pm | 19 +++++++++++++++++++ diff --git a/lib/PublicInbox/Config.pm b/lib/PublicInbox/Config.pm index 039eb445f6bbe54daadc977959e1101fae38af3c..251008a38e0e7e5b542c9675801e8d6b463a41c9 100644 --- a/lib/PublicInbox/Config.pm +++ b/lib/PublicInbox/Config.pm @@ -94,6 +94,9 @@ my ($self, $name) = @_; $self->{-ei_by_name}->{$name} //= _fill_ei($self, "extindex.$name"); } +# special case for [extindex "all"] +sub ALL { lookup_ei($_[0], 'all') } + sub each_inbox { my ($self, $cb, @arg) = @_; # may auto-vivify if config file is non-existent: diff --git a/lib/PublicInbox/Inbox.pm b/lib/PublicInbox/Inbox.pm index a1a072adc829eed9431cf4248a12915a7c20f313..5a22e40d086acf5d0b96333aa23a907328a28216 100644 --- a/lib/PublicInbox/Inbox.pm +++ b/lib/PublicInbox/Inbox.pm @@ -429,4 +429,6 @@ } sub uidvalidity { $_[0]->{uidvalidity} //= $_[0]->mm->created_at } +sub eidx_key { $_[0]->{newsgroup} // $_[0]->{inboxdir} } + 1; diff --git a/lib/PublicInbox/InboxWritable.pm b/lib/PublicInbox/InboxWritable.pm index d3c255c774604371eb2a682db0df108f77b3d8eb..e97c7e2d7588b903a0917c803cc8f0cde4b81a91 100644 --- a/lib/PublicInbox/InboxWritable.pm +++ b/lib/PublicInbox/InboxWritable.pm @@ -319,6 +319,4 @@ } $latest; } -sub eidx_key { $_[0]->{newsgroup} // $_[0]->{inboxdir} } - 1; diff --git a/lib/PublicInbox/ManifestJsGz.pm b/lib/PublicInbox/ManifestJsGz.pm index 3b436827b1554599be348f793e0b91a3ac45137a..2c4a231d9164157bb88a008bd6a02ddea3727710 100644 --- a/lib/PublicInbox/ManifestJsGz.pm +++ b/lib/PublicInbox/ManifestJsGz.pm @@ -21,6 +21,14 @@ # SUPER calls PublicInbox::WwwListing::url_regexp $ctx->SUPER::url_regexp('publicInbox.grokManifest', 'match=domain'); } +sub inject_entry ($$$;$) { + my ($ctx, $url_path, $ent, $git_dir) = @_; + $ctx->{-abs2urlpath}->{$git_dir // delete $ent->{git_dir}} = $url_path; + my $modified = $ent->{modified}; + $ctx->{-mtime} = $modified if $modified > ($ctx->{-mtime} // 0); + $ctx->{manifest}->{$url_path} = $ent; +} + sub manifest_add ($$;$$) { my ($ctx, $ibx, $epoch, $default_desc) = @_; my $url_path = "/$ibx->{name}"; @@ -32,15 +40,10 @@ } else { $git = $ibx->git; } my $ent = $git->manifest_entry($epoch, $default_desc) or return; - $ctx->{-abs2urlpath}->{$git->{git_dir}} = $url_path; - my $modified = $ent->{modified}; - if ($modified > ($ctx->{-mtime} // 0)) { - $ctx->{-mtime} = $modified; - } - $ctx->{manifest}->{$url_path} = $ent; + inject_entry($ctx, $url_path, $ent, $git->{git_dir}); } -sub ibx_entry { +sub slow_manifest_add ($$) { my ($ctx, $ibx) = @_; eval { if (defined(my $max = $ibx->max_git_epoch)) { @@ -52,6 +55,28 @@ } else { manifest_add($ctx, $ibx); } }; +} + +sub eidx_manifest_add ($$$) { + my ($ctx, $ALL, $ibx) = @_; + if (my $data = $ALL->misc->inbox_data($ibx)) { + $data = $json->decode($data); + while (my ($url_path, $ent) = each %$data) { + inject_entry($ctx, $url_path, $ent); + } + } else { + warn "E: `${\$ibx->eidx_key}' not indexed by $ALL->{topdir}\n"; + } +} + +sub ibx_entry { + my ($ctx, $ibx) = @_; + my $ALL = $ctx->{www}->{pi_config}->ALL; + if ($ALL) { + eidx_manifest_add($ctx, $ALL, $ibx); + } else { + slow_manifest_add($ctx, $ibx); + } warn "E: $@" if $@; } diff --git a/lib/PublicInbox/MiscSearch.pm b/lib/PublicInbox/MiscSearch.pm index 8beb8349fea839a4a132298ec9001a6c1758fb52..5a44d751ce920b6da603fc095c83644d3ae9fcf6 100644 --- a/lib/PublicInbox/MiscSearch.pm +++ b/lib/PublicInbox/MiscSearch.pm @@ -76,4 +76,23 @@ $opt->{relevance} = 1 unless exists $opt->{relevance}; retry_reopen($self, \&misc_enquire_once, [ $self, $qr, $opt ]); } +sub ibx_data_once { + my ($self, $ibx) = @{$_[0]}; + my $xdb = $self->{xdb}; + my $eidx_key = $ibx->eidx_key; # may be {inboxdir}, so private + my $head = $xdb->postlist_begin('Q'.$eidx_key); + my $tail = $xdb->postlist_end('Q'.$eidx_key); + if ($head != $tail) { + my $doc = $xdb->get_document($head->get_docid); + $doc->get_data; + } else { + undef; + } +} + +sub inbox_data { + my ($self, $ibx) = @_; + retry_reopen($self, \&ibx_data_once, [ $self, $ibx ]); +} + 1;