X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FNNTP.pm;h=cc6534b99810489f96c1f94176e0ec4ddb7189c2;hb=24e103d37b423c1c718e7f0f6285419005a98be5;hp=88fe2bb03a0797e424224771d8eab5e2c411784f;hpb=f429713369050203856b2b93dbdc91c258945308;p=public-inbox.git diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm index 88fe2bb0..cc6534b9 100644 --- a/lib/PublicInbox/NNTP.pm +++ b/lib/PublicInbox/NNTP.pm @@ -31,9 +31,9 @@ use Errno qw(EAGAIN); my $ONE_MSGID = qr/\A$MID_EXTRACT\z/; my @OVERVIEW = qw(Subject From Date Message-ID References); my $OVERVIEW_FMT = join(":\r\n", @OVERVIEW, qw(Bytes Lines), '') . - "Xref:full\r\n"; + "Xref:full\r\n."; my $LIST_HEADERS = join("\r\n", @OVERVIEW, - qw(:bytes :lines Xref To Cc)) . "\r\n"; + qw(:bytes :lines Xref To Cc)) . "\r\n."; my $CAPABILITIES = <<""; 101 Capability list:\r VERSION 2\r @@ -53,7 +53,7 @@ sub new ($$$) { my $wbuf; if ($sock->can('accept_SSL') && !$sock->accept_SSL) { return CORE::close($sock) if $! != EAGAIN; - $ev = PublicInbox::TLS::epollbit(); + $ev = PublicInbox::TLS::epollbit() or return CORE::close($sock); $wbuf = [ \&PublicInbox::DS::accept_tls_step, \&greet ]; } $self->SUPER::new($sock, $ev | EPOLLONESHOT); @@ -92,8 +92,7 @@ sub process_line ($$) { err($self, 'error from: %s (%s)', $l, $err); $res = '503 program fault - command not performed'; } - return 0 unless defined $res; - res($self, $res); + defined($res) ? res($self, $res) : 0; } # The keyword argument is not used (rfc3977 5.2.2) @@ -109,9 +108,7 @@ sub cmd_capabilities ($;$) { sub cmd_mode ($$) { my ($self, $arg) = @_; - $arg = uc $arg; - return r501 unless $arg eq 'READER'; - '201 Posting prohibited'; + uc($arg) eq 'READER' ? '201 Posting prohibited' : r501; } sub cmd_slave ($) { '202 slave status noted' } @@ -120,46 +117,66 @@ sub cmd_xgtitle ($;$) { my ($self, $wildmat) = @_; more($self, '282 list of groups and descriptions follows'); list_newsgroups($self, $wildmat); - '.' } -sub list_overview_fmt ($) { - my ($self) = @_; - $self->msg_more($OVERVIEW_FMT); -} +sub list_overview_fmt ($) { $OVERVIEW_FMT } -sub list_headers ($;$) { - my ($self) = @_; - $self->msg_more($LIST_HEADERS); +sub list_headers ($;$) { $LIST_HEADERS } + +sub list_active_i { # "LIST ACTIVE" and also just "LIST" (no args) + my ($self, $groupnames) = @_; + my @window = splice(@$groupnames, 0, 100) or return 0; + my $ibx; + my $groups = $self->{nntpd}->{pi_config}->{-by_newsgroup}; + for my $ngname (@window) { + $ibx = $groups->{$ngname} and group_line($self, $ibx); + } + scalar(@$groupnames); # continue if there's more } -sub list_active ($;$) { +sub list_active ($;$) { # called by cmd_list my ($self, $wildmat) = @_; wildmat2re($wildmat); - foreach my $ng (@{$self->{nntpd}->{grouplist}}) { - $ng->{newsgroup} =~ $wildmat or next; - group_line($self, $ng); + long_response($self, \&list_active_i, [ + grep(/$wildmat/, @{$self->{nntpd}->{groupnames}}) ]); +} + +sub list_active_times_i { + my ($self, $groupnames) = @_; + my @window = splice(@$groupnames, 0, 100) or return 0; + my $groups = $self->{nntpd}->{pi_config}->{-by_newsgroup}; + for my $ngname (@window) { + my $ibx = $groups->{$ngname} or next; + my $c = eval { $ibx->uidvalidity } // time; + more($self, "$ngname $c <$ibx->{-primary_address}>"); } + scalar(@$groupnames); # continue if there's more } -sub list_active_times ($;$) { +sub list_active_times ($;$) { # called by cmd_list my ($self, $wildmat) = @_; wildmat2re($wildmat); - foreach my $ng (@{$self->{nntpd}->{grouplist}}) { - $ng->{newsgroup} =~ $wildmat or next; - my $c = eval { $ng->mm->created_at } || time; - more($self, "$ng->{newsgroup} $c $ng->{-primary_address}"); + long_response($self, \&list_active_times_i, [ + grep(/$wildmat/, @{$self->{nntpd}->{groupnames}}) ]); +} + +sub list_newsgroups_i { + my ($self, $groupnames) = @_; + my @window = splice(@$groupnames, 0, 100) or return 0; + my $groups = $self->{nntpd}->{pi_config}->{-by_newsgroup}; + my $ibx; + for my $ngname (@window) { + $ibx = $groups->{$ngname} and + more($self, "$ngname ".$ibx->description); } + scalar(@$groupnames); # continue if there's more } -sub list_newsgroups ($;$) { +sub list_newsgroups ($;$) { # called by cmd_list my ($self, $wildmat) = @_; wildmat2re($wildmat); - foreach my $ng (@{$self->{nntpd}->{grouplist}}) { - $ng->{newsgroup} =~ $wildmat or next; - my $d = $ng->description; - more($self, "$ng->{newsgroup} $d"); - } + long_response($self, \&list_newsgroups_i, [ + grep(/$wildmat/, @{$self->{nntpd}->{groupnames}}) ]); } # LIST SUBSCRIPTIONS, DISTRIB.PATS are not supported @@ -168,6 +185,7 @@ sub cmd_list ($;$$) { if (scalar @args) { my $arg = shift @args; $arg =~ tr/A-Z./a-z_/; + my $ret = $arg eq 'active'; $arg = "list_$arg"; $arg = $self->can($arg); return r501 unless $arg && args_ok($arg, scalar @args); @@ -175,11 +193,9 @@ sub cmd_list ($;$$) { $arg->($self, @args); } else { more($self, '215 list of newsgroups follows'); - foreach my $ng (@{$self->{nntpd}->{grouplist}}) { - group_line($self, $ng); - } + long_response($self, \&list_active_i, [ # copy array + @{$self->{nntpd}->{groupnames}} ]); } - '.' } sub listgroup_range_i { @@ -244,7 +260,20 @@ sub parse_time ($$;$) { sub group_line ($$) { my ($self, $ng) = @_; my ($min, $max) = $ng->mm->minmax; - more($self, "$ng->{newsgroup} $max $min n") if defined $min && defined $max; + more($self, "$ng->{newsgroup} $max $min n"); +} + +sub newgroups_i { + my ($self, $ts, $i, $groupnames) = @_; + my $end = $$i + 100; + my $groups = $self->{nntpd}->{pi_config}->{-by_newsgroup}; + while ($$i < $end) { + my $ngname = $groupnames->[$$i++] // return; + my $ibx = $groups->{$ngname} or next; # expired on reload + next unless (eval { $ibx->uidvalidity } // 0) > $ts; + group_line($self, $ibx); + } + 1; } sub cmd_newgroups ($$$;$$) { @@ -254,12 +283,8 @@ sub cmd_newgroups ($$$;$$) { # TODO dists more($self, '231 list of new newsgroups follows'); - foreach my $ng (@{$self->{nntpd}->{grouplist}}) { - my $c = eval { $ng->mm->created_at } || 0; - next unless $c > $ts; - group_line($self, $ng); - } - '.' + long_response($self, \&newgroups_i, $ts, \(my $i = 0), + $self->{nntpd}->{groupnames}); } sub wildmat2re (;$) { @@ -294,23 +319,28 @@ sub ngpat2re (;$) { } sub newnews_i { - my ($self, $overs, $ts, $prev) = @_; - my $over = $overs->[0]; - my $msgs = $over->query_ts($ts, $$prev); - if (scalar @$msgs) { - more($self, '<' . - join(">\r\n<", map { $_->{mid} } @$msgs ). - '>'); - $$prev = $msgs->[-1]->{num}; - } else { - shift @$overs; - if (@$overs) { # continue onto next newsgroup - $$prev = 0; - return 1; - } else { # break out of the long response. - return; + my ($self, $names, $ts, $prev) = @_; + my $ngname = $names->[0]; + if (my $ibx = $self->{nntpd}->{groups}->{$ngname}) { + if (my $over = $ibx->over) { + my $msgs = $over->query_ts($ts, $$prev); + if (scalar @$msgs) { + more($self, '<' . + join(">\r\n<", + map { $_->{mid} } @$msgs ) . + '>'); + $$prev = $msgs->[-1]->{num}; + return 1; # continue on current group + } } } + shift @$names; + if (@$names) { # continue onto next newsgroup + $$prev = 0; + 1; + } else { # all done, break out of the long_response + undef; + } } sub cmd_newnews ($$$$;$$) { @@ -321,17 +351,11 @@ sub cmd_newnews ($$$$;$$) { my ($keep, $skip) = split('!', $newsgroups, 2); ngpat2re($keep); ngpat2re($skip); - my @overs; - foreach my $ng (@{$self->{nntpd}->{grouplist}}) { - $ng->{newsgroup} =~ $keep or next; - $ng->{newsgroup} =~ $skip and next; - my $over = $ng->over or next; - push @overs, $over; - }; - return '.' unless @overs; - + my @names = grep(!/$skip/, grep(/$keep/, + @{$self->{nntpd}->{groupnames}})); + return '.' unless scalar(@names); my $prev = 0; - long_response($self, \&newnews_i, \@overs, $ts, \$prev); + long_response($self, \&newnews_i, \@names, $ts, \$prev); } sub cmd_group ($$) { @@ -343,8 +367,6 @@ sub cmd_group ($$) { $self->{ng} = $ng; my ($min, $max) = $ng->mm->minmax; - $min ||= 0; - $max ||= 0; $self->{article} = $min; my $est_size = $max - $min; "211 $est_size $min $max $group"; @@ -395,18 +417,26 @@ sub header_append ($$$) { $hdr->header_set($k, @v, $v); } -sub xref ($$$$) { - my ($self, $ng, $n, $mid) = @_; - my $ret = $self->{nntpd}->{servername} . " $ng->{newsgroup}:$n"; - - # num_for is pretty cheap and sometimes we'll lookup the existence - # of an article without getting even the OVER info. In other words, - # I'm not sure if its worth optimizing by scanning To:/Cc: and - # PublicInbox::ExtMsg on the PSGI end is just as expensive - foreach my $other (@{$self->{nntpd}->{grouplist}}) { - next if $ng eq $other; - my $num = eval { $other->mm->num_for($mid) } or next; - $ret .= " $other->{newsgroup}:$num"; +sub xref ($$$) { + my ($self, $cur_ibx, $smsg) = @_; + my $nntpd = $self->{nntpd}; + my $cur_ngname = $cur_ibx->{newsgroup}; + my $ret = "$nntpd->{servername} $cur_ngname:$smsg->{num}"; + if (my $ALL = $nntpd->{pi_config}->ALL) { + if (my $ary = $ALL->nntp_xref_for($cur_ibx, $smsg)) { + $ret .= join(' ', '', @$ary) if scalar(@$ary); + } + # better off wrong than slow if there's thousands of groups, + # so no fallback to the slow path below: + } else { # slow path + my $mid = $smsg->{mid}; + my $groups = $nntpd->{pi_config}->{-by_newsgroup}; + for my $xngname (@{$nntpd->{groupnames}}) { + next if $cur_ngname eq $xngname; + my $xibx = $groups->{$xngname} or next; + my $num = eval { $xibx->mm->num_for($mid) } or next; + $ret .= " $xngname:$num"; + } } $ret; } @@ -415,10 +445,6 @@ sub set_nntp_headers ($$) { my ($hdr, $smsg) = @_; my ($mid) = $smsg->{mid}; - # why? leafnode requires a Path: header for some inexplicable - # reason. We'll fake the shortest one possible. - $hdr->header_set('Path', 'y'); - # leafnode (and maybe other NNTP clients) have trouble dealing # with v2 messages which have multiple Message-IDs (either due # to our own content-based dedupe or buggy git-send-email versions). @@ -432,12 +458,21 @@ sub set_nntp_headers ($$) { $hdr->header_set('X-Alt-Message-ID', @alt); } - # clobber some + # clobber some existing headers my $ibx = $smsg->{-ibx}; - my $xref = xref($smsg->{nntp}, $ibx, $smsg->{num}, $mid); + my $xref = xref($smsg->{nntp}, $ibx, $smsg); $hdr->header_set('Xref', $xref); - $xref =~ s/:[0-9]+//g; - $hdr->header_set('Newsgroups', (split(/ /, $xref, 2))[1]); + + # RFC 5536 3.1.4 + my ($server_name, $newsgroups) = split(/ /, $xref, 2); + $newsgroups =~ s/:[0-9]+\b//g; # drop NNTP article numbers + $newsgroups =~ tr/ /,/; + $hdr->header_set('Newsgroups', $newsgroups); + + # *something* here is required for leafnode, try to follow + # RFC 5536 3.1.5... + $hdr->header_set('Path', $server_name . '!not-for-mail'); + header_append($hdr, 'List-Post', "{-primary_address}>"); if (my $url = $ibx->base_url) { $mid = mid_escape($mid); @@ -706,12 +741,12 @@ sub mid_lookup ($$) { sub xref_range_i { my ($self, $beg, $end) = @_; my $ng = $self->{ng}; - my $r = $ng->mm->msg_range($beg, $end); - @$r or return; + my $msgs = $ng->over->query_xover($$beg, $end); + scalar(@$msgs) or return; + $$beg = $msgs->[-1]->{num} + 1; more($self, join("\r\n", map { - my $num = $_->[0]; - "$num ".xref($self, $ng, $num, $_->[1]); - } @$r)); + "$_->{num} ".xref($self, $ng, $_); + } @$msgs)); 1; } @@ -722,8 +757,9 @@ sub hdr_xref ($$$) { # optimize XHDR Xref [range] for rtin my $mid = $1; my ($ng, $n) = mid_lookup($self, $mid); return r430 unless $n; + my $smsg = $ng->over->get_art($n) or return; hdr_mid_response($self, $xhdr, $ng, $n, $range, - xref($self, $ng, $n, $mid)); + xref($self, $ng, $smsg)); } else { # numeric range $range = $self->{article} unless defined $range; my $r = get_range($self, $range); @@ -854,11 +890,11 @@ sub cmd_xrover ($;$) { long_response($self, \&xrover_i, @$r); } -sub over_line ($$$$) { - my ($self, $ng, $num, $smsg) = @_; +sub over_line ($$$) { + my ($self, $ng, $smsg) = @_; # n.b. field access and procedural calls can be # 10%-15% faster than OO method calls: - my $s = join("\t", $num, + my $s = join("\t", $smsg->{num}, $smsg->{subject}, $smsg->{from}, PublicInbox::Smsg::date($smsg), @@ -866,7 +902,7 @@ sub over_line ($$$$) { $smsg->{references}, $smsg->{bytes}, $smsg->{lines}, - "Xref: " . xref($self, $ng, $num, $smsg->{mid})); + "Xref: " . xref($self, $ng, $smsg)); utf8::encode($s); $s } @@ -881,8 +917,8 @@ sub cmd_over ($;$) { # Only set article number column if it's the current group my $self_ng = $self->{ng}; - $n = 0 if (!$self_ng || $self_ng ne $ng); - more($self, over_line($self, $ng, $n, $smsg)); + $smsg->{num} = 0 if (!$self_ng || $self_ng ne $ng); + more($self, over_line($self, $ng, $smsg)); '.'; } else { cmd_xover($self, $range); @@ -897,7 +933,7 @@ sub xover_i { # OVERVIEW.FMT more($self, join("\r\n", map { - over_line($self, $ng, $_->{num}, $_); + over_line($self, $ng, $_); } @$msgs)); $$beg = $msgs->[-1]->{num} + 1; }