X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FNNTP.pm;h=eb2c0b38c445025dec375e4dd8f443a35ced59e5;hb=f5604d35bbe02ad72a4e8d5b2189ea31acf1d7b5;hp=be3bddc3f5d2a230642b4bf2dc878f7b6c5aec73;hpb=8fd41797b24736dfdccfacc5acc473234a29758a;p=public-inbox.git diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm index be3bddc3..eb2c0b38 100644 --- a/lib/PublicInbox/NNTP.pm +++ b/lib/PublicInbox/NNTP.pm @@ -2,17 +2,21 @@ # License: AGPL-3.0+ # # Each instance of this represents a NNTP client socket +# fields: +# nntpd: PublicInbox::NNTPD ref +# article: per-session current article number +# ng: PublicInbox::Inbox ref +# long_cb: long_response private data package PublicInbox::NNTP; use strict; -use warnings; -use base qw(PublicInbox::DS); -use fields qw(nntpd article ng long_cb); +use parent qw(PublicInbox::DS); use PublicInbox::MID qw(mid_escape $MID_EXTRACT); use PublicInbox::Eml; use POSIX qw(strftime); use PublicInbox::DS qw(now); use Digest::SHA qw(sha1_hex); use Time::Local qw(timegm timelocal); +use PublicInbox::GitAsyncCat; use constant { LINE_MAX => 512, # RFC 977 section 2.3 r501 => '501 command syntax error', @@ -44,16 +48,15 @@ sub greet ($) { $_[0]->write($_[0]->{nntpd}->{greet}) }; sub new ($$$) { my ($class, $sock, $nntpd) = @_; - my $self = fields::new($class); + my $self = bless { nntpd => $nntpd }, $class; my $ev = EPOLLIN; my $wbuf; if ($sock->can('accept_SSL') && !$sock->accept_SSL) { return CORE::close($sock) if $! != EAGAIN; - $ev = PublicInbox::TLS::epollbit(); + $ev = PublicInbox::TLS::epollbit() or return CORE::close($sock); $wbuf = [ \&PublicInbox::DS::accept_tls_step, \&greet ]; } $self->SUPER::new($sock, $ev | EPOLLONESHOT); - $self->{nntpd} = $nntpd; if ($wbuf) { $self->{wbuf} = $wbuf; } else { @@ -133,29 +136,29 @@ sub list_headers ($;$) { sub list_active ($;$) { my ($self, $wildmat) = @_; wildmat2re($wildmat); - foreach my $ng (@{$self->{nntpd}->{grouplist}}) { - $ng->{newsgroup} =~ $wildmat or next; - group_line($self, $ng); + my $groups = $self->{nntpd}->{groups}; + for my $ngname (grep(/$wildmat/, @{$self->{nntpd}->{groupnames}})) { + group_line($self, $groups->{$ngname}); } } sub list_active_times ($;$) { my ($self, $wildmat) = @_; wildmat2re($wildmat); - foreach my $ng (@{$self->{nntpd}->{grouplist}}) { - $ng->{newsgroup} =~ $wildmat or next; - my $c = eval { $ng->mm->created_at } || time; - more($self, "$ng->{newsgroup} $c $ng->{-primary_address}"); + my $groups = $self->{nntpd}->{groups}; + for my $ngname (grep(/$wildmat/, @{$self->{nntpd}->{groupnames}})) { + my $ibx = $groups->{$ngname}; + my $c = eval { $ibx->uidvalidity } // time; + more($self, "$ngname $c <$ibx->{-primary_address}>"); } } sub list_newsgroups ($;$) { my ($self, $wildmat) = @_; wildmat2re($wildmat); - foreach my $ng (@{$self->{nntpd}->{grouplist}}) { - $ng->{newsgroup} =~ $wildmat or next; - my $d = $ng->description; - more($self, "$ng->{newsgroup} $d"); + my $groups = $self->{nntpd}->{groups}; + for my $ngname (grep(/$wildmat/, @{$self->{nntpd}->{groupnames}})) { + more($self, "$ngname ".$groups->{$ngname}->description); } } @@ -241,7 +244,7 @@ sub parse_time ($$;$) { sub group_line ($$) { my ($self, $ng) = @_; my ($min, $max) = $ng->mm->minmax; - more($self, "$ng->{newsgroup} $max $min n") if defined $min && defined $max; + more($self, "$ng->{newsgroup} $max $min n"); } sub cmd_newgroups ($$$;$$) { @@ -252,7 +255,7 @@ sub cmd_newgroups ($$$;$$) { # TODO dists more($self, '231 list of new newsgroups follows'); foreach my $ng (@{$self->{nntpd}->{grouplist}}) { - my $c = eval { $ng->mm->created_at } || 0; + my $c = eval { $ng->uidvalidity } // 0; next unless $c > $ts; group_line($self, $ng); } @@ -291,23 +294,28 @@ sub ngpat2re (;$) { } sub newnews_i { - my ($self, $overs, $ts, $prev) = @_; - my $over = $overs->[0]; - my $msgs = $over->query_ts($ts, $$prev); - if (scalar @$msgs) { - more($self, '<' . - join(">\r\n<", map { $_->{mid} } @$msgs ). - '>'); - $$prev = $msgs->[-1]->{num}; - } else { - shift @$overs; - if (@$overs) { # continue onto next newsgroup - $$prev = 0; - return 1; - } else { # break out of the long response. - return; + my ($self, $names, $ts, $prev) = @_; + my $ngname = $names->[0]; + if (my $ibx = $self->{nntpd}->{groups}->{$ngname}) { + if (my $over = $ibx->over) { + my $msgs = $over->query_ts($ts, $$prev); + if (scalar @$msgs) { + more($self, '<' . + join(">\r\n<", + map { $_->{mid} } @$msgs ) . + '>'); + $$prev = $msgs->[-1]->{num}; + return 1; # continue on current group + } } } + shift @$names; + if (@$names) { # continue onto next newsgroup + $$prev = 0; + 1; + } else { # all done, break out of the long_response + undef; + } } sub cmd_newnews ($$$$;$$) { @@ -318,17 +326,11 @@ sub cmd_newnews ($$$$;$$) { my ($keep, $skip) = split('!', $newsgroups, 2); ngpat2re($keep); ngpat2re($skip); - my @overs; - foreach my $ng (@{$self->{nntpd}->{grouplist}}) { - $ng->{newsgroup} =~ $keep or next; - $ng->{newsgroup} =~ $skip and next; - my $over = $ng->over or next; - push @overs, $over; - }; - return '.' unless @overs; - + my @names = grep(!/$skip/, grep(/$keep/, + @{$self->{nntpd}->{groupnames}})); + return '.' unless scalar(@names); my $prev = 0; - long_response($self, \&newnews_i, \@overs, $ts, \$prev); + long_response($self, \&newnews_i, \@names, $ts, \$prev); } sub cmd_group ($$) { @@ -340,8 +342,6 @@ sub cmd_group ($$) { $self->{ng} = $ng; my ($min, $max) = $ng->mm->minmax; - $min ||= 0; - $max ||= 0; $self->{article} = $min; my $est_size = $max - $min; "211 $est_size $min $max $group"; @@ -408,12 +408,9 @@ sub xref ($$$$) { $ret; } -sub set_nntp_headers ($$$$$) { - my ($self, $hdr, $ng, $n, $mid) = @_; - - # why? leafnode requires a Path: header for some inexplicable - # reason. We'll fake the shortest one possible. - $hdr->header_set('Path', 'y'); +sub set_nntp_headers ($$) { + my ($hdr, $smsg) = @_; + my ($mid) = $smsg->{mid}; # leafnode (and maybe other NNTP clients) have trouble dealing # with v2 messages which have multiple Message-IDs (either due @@ -428,13 +425,23 @@ sub set_nntp_headers ($$$$$) { $hdr->header_set('X-Alt-Message-ID', @alt); } - # clobber some - my $xref = xref($self, $ng, $n, $mid); + # clobber some existing headers + my $ibx = $smsg->{-ibx}; + my $xref = xref($smsg->{nntp}, $ibx, $smsg->{num}, $mid); $hdr->header_set('Xref', $xref); - $xref =~ s/:[0-9]+//g; - $hdr->header_set('Newsgroups', (split(/ /, $xref, 2))[1]); - header_append($hdr, 'List-Post', "{-primary_address}>"); - if (my $url = $ng->base_url) { + + # RFC 5536 3.1.4 + my ($server_name, $newsgroups) = split(/ /, $xref, 2); + $newsgroups =~ s/:[0-9]+\b//g; # drop NNTP article numbers + $newsgroups =~ tr/ /,/; + $hdr->header_set('Newsgroups', $newsgroups); + + # *something* here is required for leafnode, try to follow + # RFC 5536 3.1.5... + $hdr->header_set('Path', $server_name . '!not-for-mail'); + + header_append($hdr, 'List-Post', "{-primary_address}>"); + if (my $url = $ibx->base_url) { $mid = mid_escape($mid); header_append($hdr, 'Archived-At', "<$url$mid/>"); header_append($hdr, 'List-Archive', "<$url>"); @@ -442,7 +449,7 @@ sub set_nntp_headers ($$$$$) { } sub art_lookup ($$$) { - my ($self, $art, $set_headers) = @_; + my ($self, $art, $code) = @_; my $ng = $self->{ng}; my ($n, $mid); my $err; @@ -478,13 +485,18 @@ find_mid: } found: my $smsg = $ng->over->get_art($n) or return $err; - my $msg = $ng->msg_by_smsg($smsg) or return $err; - - # PublicInbox::Eml->new will modify $msg in-place, so what's - # left is the body and we won't need to call ->body(), later - my $hdr = PublicInbox::Eml->new($msg)->header_obj; - set_nntp_headers($self, $hdr, $ng, $n, $mid) if $set_headers; - [ $n, $mid, $msg, $hdr ]; + $smsg->{-ibx} = $ng; + if ($code == 223) { # STAT + set_art($self, $n); + "223 $n <$smsg->{mid}> article retrieved - " . + "request text separately"; + } else { # HEAD | BODY | ARTICLE + $smsg->{nntp} = $self; + $smsg->{nntp_code} = $code; + set_art($self, $art); + # this dereferences to `undef' + ${git_async_cat($ng->git, $smsg->{blob}, \&blob_cb, $smsg)}; + } } sub msg_body_write ($$) { @@ -495,7 +507,6 @@ sub msg_body_write ($$) { $$msg =~ s/(?msg_more($$msg); - '.' } sub set_art { @@ -503,60 +514,73 @@ sub set_art { $self->{article} = $art if defined $art && $art =~ /\A[0-9]+\z/; } -sub msg_hdr_write ($$$) { - my ($self, $hdr, $body_follows) = @_; - $hdr = $hdr->as_string; +sub msg_hdr_write ($$) { + my ($eml, $smsg) = @_; + set_nntp_headers($eml, $smsg); + + my $hdr = $eml->{hdr} // \(my $x = ''); # fixup old bug from import (pre-a0c07cba0e5d8b6a) - $hdr =~ s/\A[\r\n]*From [^\r\n]*\r?\n//s; - utf8::encode($hdr); - $hdr =~ s/(?msg_more($hdr); + $$hdr =~ s/^(Message-ID:)[ \t]*\r\n[ \t]+([^\r]+)\r\n/$1 $2\r\n/igsm; + $smsg->{nntp}->msg_more($$hdr); +} + +sub blob_cb { # called by git->cat_async via git_async_cat + my ($bref, $oid, $type, $size, $smsg) = @_; + my $self = $smsg->{nntp}; + my $code = $smsg->{nntp_code}; + if (!defined($oid)) { + # it's possible to have TOCTOU if an admin runs + # public-inbox-(edit|purge), just move onto the next message + warn "E: $smsg->{blob} missing in $smsg->{-ibx}->{inboxdir}\n"; + return $self->requeue; + } elsif ($smsg->{blob} ne $oid) { + $self->close; + die "BUG: $smsg->{blob} != $oid"; + } + my $r = "$code $smsg->{num} <$smsg->{mid}> article retrieved - "; + my $eml = PublicInbox::Eml->new($bref); + if ($code == 220) { + more($self, $r .= 'head and body follow'); + msg_hdr_write($eml, $smsg); + $self->msg_more("\r\n"); + msg_body_write($self, $bref); + } elsif ($code == 221) { + more($self, $r .= 'head follows'); + msg_hdr_write($eml, $smsg); + } elsif ($code == 222) { + more($self, $r .= 'body follows'); + msg_body_write($self, $bref); + } else { + $self->close; + die "BUG: bad code: $r"; + } + $self->write(\".\r\n"); # flushes (includes ->zflush) + $self->requeue; } sub cmd_article ($;$) { my ($self, $art) = @_; - my $r = art_lookup($self, $art, 1); - return $r unless ref $r; - my ($n, $mid, $msg, $hdr) = @$r; - set_art($self, $art); - more($self, "220 $n <$mid> article retrieved - head and body follow"); - msg_hdr_write($self, $hdr, 1); - msg_body_write($self, $msg); + art_lookup($self, $art, 220); } sub cmd_head ($;$) { my ($self, $art) = @_; - my $r = art_lookup($self, $art, 2); - return $r unless ref $r; - my ($n, $mid, undef, $hdr) = @$r; - set_art($self, $art); - more($self, "221 $n <$mid> article retrieved - head follows"); - msg_hdr_write($self, $hdr, 0); - '.' + art_lookup($self, $art, 221); } sub cmd_body ($;$) { my ($self, $art) = @_; - my $r = art_lookup($self, $art, 0); - return $r unless ref $r; - my ($n, $mid, $msg) = @$r; - set_art($self, $art); - more($self, "222 $n <$mid> article retrieved - body follows"); - msg_body_write($self, $msg); + art_lookup($self, $art, 222); } sub cmd_stat ($;$) { my ($self, $art) = @_; - my $r = art_lookup($self, $art, 0); - return $r unless ref $r; - my ($n, $mid) = @$r; - set_art($self, $art); - "223 $n <$mid> article retrieved - request text separately"; + art_lookup($self, $art, 223); # art may be msgid } sub cmd_ihave ($) { '435 article not wanted - do not send it' } @@ -956,38 +980,35 @@ sub out ($$;@) { sub event_step { my ($self) = @_; - return unless $self->flush_write && $self->{sock}; + return unless $self->flush_write && $self->{sock} && !$self->{long_cb}; $self->update_idle_time; # only read more requests if we've drained the write buffer, # otherwise we can be buffering infinitely w/o backpressure - my $rbuf = $self->{rbuf} // (\(my $x = '')); - my $r = 1; - - if (index($$rbuf, "\n") < 0) { - my $off = bytes::length($$rbuf); - $r = $self->do_read($rbuf, LINE_MAX, $off) or return; - } - while ($r > 0 && $$rbuf =~ s/\A[ \t]*([^\n]*?)\r?\n//) { - my $line = $1; - return $self->close if $line =~ /[[:cntrl:]]/s; - my $t0 = now(); - my $fd = fileno($self->{sock}); - $r = eval { process_line($self, $line) }; - my $pending = $self->{wbuf} ? ' pending' : ''; - out($self, "[$fd] %s - %0.6f$pending", $line, now() - $t0); - } - + my $rbuf = $self->{rbuf} // \(my $x = ''); + my $line = index($$rbuf, "\n"); + while ($line < 0) { + return $self->close if length($$rbuf) >= LINE_MAX; + $self->do_read($rbuf, LINE_MAX, length($$rbuf)) or return; + $line = index($$rbuf, "\n"); + } + $line = substr($$rbuf, 0, $line + 1, ''); + $line =~ s/\r?\n\z//s; + return $self->close if $line =~ /[[:cntrl:]]/s; + + my $t0 = now(); + my $fd = fileno($self->{sock}); + my $r = eval { process_line($self, $line) }; + my $pending = $self->{wbuf} ? ' pending' : ''; + out($self, "[$fd] %s - %0.6f$pending", $line, now() - $t0); return $self->close if $r < 0; - my $len = bytes::length($$rbuf); - return $self->close if ($len >= LINE_MAX); $self->rbuf_idle($rbuf); $self->update_idle_time; # maybe there's more pipelined data, or we'll have # to register it for socket-readiness notifications - $self->requeue unless $self->{wbuf}; + $self->requeue unless $pending; } # for graceful shutdown in PublicInbox::Daemon: