nntp: NEWGROUPS uses long_response

We can amortize the cost of NEWGROUPS time filtering using the
long_response API.  This lets us handle hundreds/thousands of
inboxes without monopolizing the event loop for this command.

Further speedup is possible using MiscSearch, but that requires
not-yet-done indexing changes to MiscIdx.

diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm
index 8eec6b91ecba48d6ee3efaa62eda6c8c34a0dc91..cc6534b99810489f96c1f94176e0ec4ddb7189c2 100644 (file)
--- a/lib/PublicInbox/NNTP.pm
+++ b/lib/PublicInbox/NNTP.pm
@@ -263,6 +263,19 @@ sub group_line ($$) {
        more($self, "$ng->{newsgroup} $max $min n");
 }
 
+sub newgroups_i {
+       my ($self, $ts, $i, $groupnames) = @_;
+       my $end = $$i + 100;
+       my $groups = $self->{nntpd}->{pi_config}->{-by_newsgroup};
+       while ($$i < $end) {
+               my $ngname = $groupnames->[$$i++] // return;
+               my $ibx = $groups->{$ngname} or next; # expired on reload
+               next unless (eval { $ibx->uidvalidity } // 0) > $ts;
+               group_line($self, $ibx);
+       }
+       1;
+}
+
 sub cmd_newgroups ($$$;$$) {
        my ($self, $date, $time, $gmt, $dists) = @_;
        my $ts = eval { parse_time($date, $time, $gmt) };
@@ -270,12 +283,8 @@ sub cmd_newgroups ($$$;$$) {
 
        # TODO dists
        more($self, '231 list of new newsgroups follows');
-       foreach my $ng (@{$self->{nntpd}->{grouplist}}) {
-               my $c = eval { $ng->uidvalidity } // 0;
-               next unless $c > $ts;
-               group_line($self, $ng);
-       }
-       '.'
+       long_response($self, \&newgroups_i, $ts, \(my $i = 0),
+                               $self->{nntpd}->{groupnames});
 }
 
 sub wildmat2re (;$) {

diff --git a/lib/PublicInbox/NNTPD.pm b/lib/PublicInbox/NNTPD.pm
index 4de1944b27206c36e7afc6bce44353de532843ea..33bc5fda7a9a936e0fc95c28a14d91113d3bda31 100644 (file)
--- a/lib/PublicInbox/NNTPD.pm
+++ b/lib/PublicInbox/NNTPD.pm
@@ -24,7 +24,6 @@ sub new {
                groups => {},
                err => \*STDERR,
                out => \*STDOUT,
-               grouplist => [],
                pi_config => $pi_config,
                servername => $name,
                greet => \"201 $name ready - post via email\r\n",
@@ -60,9 +59,7 @@ sub refresh_groups {
                        delete $groups->{$ngname};
                }
        });
-       my @names = sort(keys %$groups);
-       $self->{grouplist} = [ map { $groups->{$_} } @names ];
-       $self->{groupnames} = \@names;
+       $self->{groupnames} = [ sort(keys %$groups) ];
        $self->{pi_config} = $pi_config;
        # this will destroy old groups that got deleted
        $self->{groups} = $groups;

diff --git a/t/nntp.t b/t/nntp.t
index 91a2aff7a5b4506d97e1be073fd9cf73792e2514..ea2ef8763114cd79cc184380fa9d95ff3a30d4b4 100644 (file)
--- a/t/nntp.t
+++ b/t/nntp.t
@@ -112,7 +112,6 @@ use PublicInbox::Config;
        my $hdr = $mime->header_obj;
        my $mock_self = {
                nntpd => {
-                       grouplist => [],
                        servername => 'example.com',
                        pi_config => bless {}, 'PublicInbox::Config',
                },