nntp: speed up mid_lookup() using ->ALL extindex

[public-inbox.git] / lib / PublicInbox / NNTP.pm

diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm
index 8eec6b91ecba48d6ee3efaa62eda6c8c34a0dc91..7b3b1ffe98eb99fe8a00a4186c207ca70d203a9b 100644 (file)
--- a/lib/PublicInbox/NNTP.pm
+++ b/lib/PublicInbox/NNTP.pm
@@ -263,6 +263,19 @@ sub group_line ($$) {
        more($self, "$ng->{newsgroup} $max $min n");
 }
 
+sub newgroups_i {
+       my ($self, $ts, $i, $groupnames) = @_;
+       my $end = $$i + 100;
+       my $groups = $self->{nntpd}->{pi_config}->{-by_newsgroup};
+       while ($$i < $end) {
+               my $ngname = $groupnames->[$$i++] // return;
+               my $ibx = $groups->{$ngname} or next; # expired on reload
+               next unless (eval { $ibx->uidvalidity } // 0) > $ts;
+               group_line($self, $ibx);
+       }
+       1;
+}
+
 sub cmd_newgroups ($$$;$$) {
        my ($self, $date, $time, $gmt, $dists) = @_;
        my $ts = eval { parse_time($date, $time, $gmt) };
@@ -270,12 +283,8 @@ sub cmd_newgroups ($$$;$$) {
 
        # TODO dists
        more($self, '231 list of new newsgroups follows');
-       foreach my $ng (@{$self->{nntpd}->{grouplist}}) {
-               my $c = eval { $ng->uidvalidity } // 0;
-               next unless $c > $ts;
-               group_line($self, $ng);
-       }
-       '.'
+       long_response($self, \&newgroups_i, $ts, \(my $i = 0),
+                               $self->{nntpd}->{groupnames});
 }
 
 sub wildmat2re (;$) {
@@ -721,10 +730,36 @@ sub mid_lookup ($$) {
                my $n = $self_ng->mm->num_for($mid);
                return ($self_ng, $n) if defined $n;
        }
-       foreach my $ng (values %{$self->{nntpd}->{groups}}) {
-               next if defined $self_ng && $ng eq $self_ng;
-               my $n = $ng->mm->num_for($mid);
-               return ($ng, $n) if defined $n;
+       my $pi_cfg = $self->{nntpd}->{pi_config};
+       if (my $ALL = $pi_cfg->ALL) {
+               my ($id, $prev);
+               while (my $smsg = $ALL->over->next_by_mid($mid, \$id, \$prev)) {
+                       my $xr3 = $ALL->over->get_xref3($smsg->{num});
+                       if (my @x = grep(/:$smsg->{blob}\z/, @$xr3)) {
+                               my ($ngname, $xnum) = split(/:/, $x[0]);
+                               my $ibx = $pi_cfg->{-by_newsgroup}->{$ngname};
+                               return ($ibx, $xnum) if $ibx;
+                               # fall through to trying all xref3s
+                       } else {
+                               warn <<EOF;
+W: xref3 missing for <$mid> ($smsg->{blob}) in $ALL->{topdir}, -extindex bug?
+EOF
+                       }
+                       # try all xref3s
+                       for my $x (@$xr3) {
+                               my ($ngname, $xnum) = split(/:/, $x);
+                               my $ibx = $pi_cfg->{-by_newsgroup}->{$ngname};
+                               return ($ibx, $xnum) if $ibx;
+                               warn "W: `$ngname' does not exist for #$xnum\n";
+                       }
+               }
+               # no warning here, $mid is just invalid
+       } else { # slow path for non-ALL users
+               foreach my $ibx (values %{$self->{nntpd}->{groups}}) {
+                       next if defined $self_ng && $ibx eq $self_ng;
+                       my $n = $ibx->mm->num_for($mid);
+                       return ($ibx, $n) if defined $n;
+               }
        }
        (undef, undef);
 }