From d7780e12856d83b9f5c58586691e2fbe711321ed Mon Sep 17 00:00:00 2001
From: Eric Wong <e@80x24.org>
Date: Thu, 26 Aug 2021 12:33:38 +0000
Subject: [PATCH] move ->ids_after from mm to over
Since we favor ->over in WWW and IMAP, move this method to
->over to reduce open files in common cases.
This fixes the /$EXTINDEX_NAME/all.mbox.gz endpoint for extindex
entries (which may get expensive...).
---
lib/PublicInbox/Mbox.pm | 10 ++++------
lib/PublicInbox/Msgmap.pm | 11 -----------
lib/PublicInbox/NNTP.pm | 2 +-
lib/PublicInbox/Over.pm | 11 +++++++++++
t/extindex-psgi.t | 3 +++
5 files changed, 19 insertions(+), 18 deletions(-)
diff --git a/lib/PublicInbox/Mbox.pm b/lib/PublicInbox/Mbox.pm
index 844099aa..f72af26b 100644
--- a/lib/PublicInbox/Mbox.pm
+++ b/lib/PublicInbox/Mbox.pm
@@ -161,19 +161,17 @@ sub all_ids_cb {
my $smsg = $ctx->{over}->get_art($num) or next;
return $smsg;
}
- $ctx->{ids} = $ids = $ctx->{mm}->ids_after(\($ctx->{prev}));
+ $ctx->{ids} = $ids = $ctx->{over}->ids_after(\($ctx->{prev}));
} while (@$ids);
}
sub mbox_all_ids {
my ($ctx) = @_;
- my $ibx = $ctx->{ibx};
my $prev = 0;
- my $mm = $ctx->{mm} = $ibx->mm;
- my $ids = $mm->ids_after(\$prev) or return
- [404, [qw(Content-Type text/plain)], ["No results found\n"]];
- $ctx->{over} = $ibx->over or
+ $ctx->{over} = $ctx->{ibx}->over or
return PublicInbox::WWW::need($ctx, 'Overview');
+ my $ids = $ctx->{over}->ids_after(\$prev) or return
+ [404, [qw(Content-Type text/plain)], ["No results found\n"]];
$ctx->{ids} = $ids;
$ctx->{prev} = $prev;
require PublicInbox::MboxGz;
diff --git a/lib/PublicInbox/Msgmap.pm b/lib/PublicInbox/Msgmap.pm
index 16a9a476..3887a9e6 100644
--- a/lib/PublicInbox/Msgmap.pm
+++ b/lib/PublicInbox/Msgmap.pm
@@ -189,17 +189,6 @@ CREATE TABLE IF NOT EXISTS meta (
}
-# used by NNTP.pm
-sub ids_after {
- my ($self, $num) = @_;
- my $ids = $self->{dbh}->selectcol_arrayref(<<'', undef, $$num);
-SELECT num FROM msgmap WHERE num > ?
-ORDER BY num ASC LIMIT 1000
-
- $$num = $ids->[-1] if @$ids;
- $ids;
-}
-
sub msg_range {
my ($self, $beg, $end, $cols) = @_;
$cols //= 'num,mid';
diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm
index aea04c05..ea9ce183 100644
--- a/lib/PublicInbox/NNTP.pm
+++ b/lib/PublicInbox/NNTP.pm
@@ -210,7 +210,7 @@ sub listgroup_range_i {
sub listgroup_all_i {
my ($self, $num) = @_;
- my $ary = $self->{ibx}->mm(1)->ids_after($num);
+ my $ary = $self->{ibx}->over(1)->ids_after($num);
scalar(@$ary) or return;
more($self, join("\r\n", @$ary));
1;
diff --git a/lib/PublicInbox/Over.pm b/lib/PublicInbox/Over.pm
index 58fdea0e..19da056a 100644
--- a/lib/PublicInbox/Over.pm
+++ b/lib/PublicInbox/Over.pm
@@ -371,4 +371,15 @@ SELECT COUNT(*) FROM xref3 WHERE oidbin = ?
sub blob_exists { oidbin_exists($_[0], pack('H*', $_[1])) }
+# used by NNTP.pm
+sub ids_after {
+ my ($self, $num) = @_;
+ my $ids = dbh($self)->selectcol_arrayref(<<'', undef, $$num);
+SELECT num FROM over WHERE num > ?
+ORDER BY num ASC LIMIT 1000
+
+ $$num = $ids->[-1] if @$ids;
+ $ids;
+}
+
1;
diff --git a/t/extindex-psgi.t b/t/extindex-psgi.t
index b9acc979..d4761641 100644
--- a/t/extindex-psgi.t
+++ b/t/extindex-psgi.t
@@ -52,6 +52,9 @@ my $client = sub {
my $cfg = PublicInbox::Config->git_config_dump($f);
is($?, 0, 'no errors from git-config parsing');
ok($cfg->{'extindex.all.topdir'}, 'extindex.topdir defined');
+
+ $res = $cb->(GET('/all/all.mbox.gz'));
+ is($res->code, 200, 'all.mbox.gz');
};
test_psgi(sub { $www->call(@_) }, $client);
%$env = (%$env, TMPDIR => $tmpdir, PI_CONFIG => $pi_config);
--
2.50.0