nntp: use long response API for LISTGROUP

LISTGROUP can be expensive for giant groups, too.  Use the
long response API to improve fairness and prevent excessive
buffering.

diff --git a/lib/PublicInbox/Msgmap.pm b/lib/PublicInbox/Msgmap.pm
index c0fc636f67be6f69a93d281507139368c3a522e1..2f64d90c71f741cb100c3beacd124e2698260d71 100644 (file)
--- a/lib/PublicInbox/Msgmap.pm
+++ b/lib/PublicInbox/Msgmap.pm
@@ -155,25 +155,17 @@ sub create_tables {
                        'val VARCHAR(255) NOT NULL)');
 }
 
-sub each_id_batch {
-       my ($self, $cb) = @_;
+sub id_batch {
+       my ($self, $num, $cb) = @_;
        my $dbh = $self->{dbh};
-       my $n = 0;
-       my $total = 0;
-       my $nr;
        my $sth = $dbh->prepare('SELECT num FROM msgmap WHERE num > ? '.
                                'ORDER BY num ASC LIMIT 1000');
-       while (1) {
-               $sth->execute($n);
-               my $ary = $sth->fetchall_arrayref;
-               @$ary = map { $_->[0] } @$ary;
-               $nr = scalar @$ary;
-               last if $nr == 0;
-               $total += $nr;
-               $n = $ary->[-1];
-               $cb->($ary);
-       }
-       $total;
+       $sth->execute($num);
+       my $ary = $sth->fetchall_arrayref;
+       @$ary = map { $_->[0] } @$ary;
+       my $nr = scalar @$ary;
+       $cb->($ary) if $nr;
+       $nr;
 }
 
 1;

diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm
index f86c6335f5f9d5d5b53527eeb6fcb2a4d2e8f753..5d770bdbce3b8f247cc285aaffa11d2c48ea59a9 100644 (file)
--- a/lib/PublicInbox/NNTP.pm
+++ b/lib/PublicInbox/NNTP.pm
@@ -14,6 +14,7 @@ use POSIX qw(strftime);
 use Time::HiRes qw(gettimeofday tv_interval ualarm);
 use constant {
        r501 => '501 command syntax error',
+       long_response_limit => 0xffffffff,
 };
 
 my @OVERVIEW = qw(Subject From Date Message-ID References Bytes Lines);
@@ -142,13 +143,17 @@ sub cmd_listgroup {
                more($self, $res);
        }
 
-       my $ng = $self->{ng} or return '412 no newsgroup selected';
-       # Ugh this can be silly expensive for big groups
-       $ng->mm->each_id_batch(sub {
-               my ($ary) = @_;
-               more($self, join("\r\n", @$ary));
+       $self->{ng} or return '412 no newsgroup selected';
+       $self->long_response(0, long_response_limit, sub {
+               my ($i) = @_;
+               my $nr = $self->{ng}->mm->id_batch($$i, sub {
+                       my ($ary) = @_;
+                       more($self, join("\r\n", @$ary));
+               });
+
+               # -1 to adjust for implicit increment in long_response
+               $$i = $nr ? $$i + $nr - 1 : long_response_limit;
        });
-       '.'
 }
 
 sub parse_time {
@@ -245,9 +250,7 @@ sub cmd_newnews {
 
        $ts .= '..';
        my $opts = { asc => 1, limit => 1000, offset => 0 };
-
-       my $end = 0xffffffff; # would like to read 4 billion messages?
-       $self->long_response(0, $end, sub {
+       $self->long_response(0, long_response_limit, sub {
                my ($i) = @_;
                my $srch = $srch[0];
                my $res = $srch->query($ts, $opts);
@@ -262,7 +265,7 @@ sub cmd_newnews {
                        if (@srch) { # continue onto next newsgroup
                                $opts->{offset} = 0;
                        } else { # break out of the long response.
-                               $$i = $end;
+                               $$i = long_response_limit;
                        }
                }
        });