1 # Copyright (C) 2015 all contributors <meta@public-inbox.org>
2 # License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
3 package PublicInbox::NNTP;
6 use base qw(Danga::Socket);
7 use fields qw(nntpd article ng long_res);
8 use PublicInbox::Msgmap;
9 use PublicInbox::GitCatFile;
10 use PublicInbox::MID qw(mid2path);
12 use Data::Dumper qw(Dumper);
13 use POSIX qw(strftime);
14 use Time::HiRes qw(gettimeofday tv_interval ualarm);
16 r501 => '501 command syntax error',
19 my @OVERVIEW = qw(Subject From Date Message-ID References Bytes Lines);
20 my %OVERVIEW = map { $_ => 1 } @OVERVIEW;
22 # disable commands with easy DoS potential:
23 # LISTGROUP could get pretty bad, too...
24 my %DISABLED; # = map { $_ => 1 } qw(xover list_overview_fmt newnews xhdr);
27 my ($class, $sock, $nntpd) = @_;
28 my $self = fields::new($class);
29 $self->SUPER::new($sock);
30 $self->{nntpd} = $nntpd;
31 res($self, '201 server ready - post via email');
36 # returns 1 if we can continue, 0 if not due to buffered writes or disconnect
39 my ($req, @args) = split(/\s+/, $l);
43 $req = $DISABLED{$req} ? undef : *{'cmd_'.$req}{CODE};
45 return res($self, '500 command not recognized') unless $req;
47 my $res = eval { $req->($self, @args) };
49 if ($err && !$self->{closed}) {
50 chomp($l = Dumper(\$l));
51 warning('error from: ', $l, ' ', $err);
52 $res = '503 program fault - command not performed';
54 return 0 unless defined $res;
59 my ($self, $arg) = @_;
60 return r501 unless defined $arg;
62 return r501 unless $arg eq 'READER';
63 '200 reader status acknowledged';
69 '202 slave status noted';
73 my ($self, $wildmat) = @_;
74 more($self, '282 list of groups and descriptions follows');
75 list_newsgroups($self, $wildmat);
79 sub list_overview_fmt {
81 more($self, $_ . ':') foreach @OVERVIEW;
85 my ($self, $wildmat) = @_;
87 foreach my $ng (values %{$self->{nntpd}->{groups}}) {
88 $ng->{name} =~ $wildmat or next;
89 group_line($self, $ng);
93 sub list_active_times {
94 my ($self, $wildmat) = @_;
96 foreach my $ng (values %{$self->{nntpd}->{groups}}) {
97 $ng->{name} =~ $wildmat or next;
98 my $c = eval { $ng->mm->created_at } || time;
99 more($self, "$ng->{name} $c $ng->{address}");
103 sub list_newsgroups {
104 my ($self, $wildmat) = @_;
105 wildmat2re($wildmat);
106 foreach my $ng (values %{$self->{nntpd}->{groups}}) {
107 $ng->{name} =~ $wildmat or next;
108 my $d = $ng->description;
109 more($self, "$ng->{name} $d");
113 # LIST SUBSCRIPTIONS not supported
115 my ($self, $arg, $wildmat, @x) = @_;
120 return '503 function not performed' if $DISABLED{$arg};
125 return r501 unless $arg;
126 more($self, '215 information follows');
127 $arg->($self, $wildmat, @x);
129 more($self, '215 list of newsgroups follows');
130 foreach my $ng (values %{$self->{nntpd}->{groups}}) {
131 group_line($self, $ng);
138 my ($self, $group) = @_;
139 if (defined $group) {
140 my $res = cmd_group($self, $group);
141 return $res if ($res !~ /\A211 /);
145 my $ng = $self->{ng} or return '412 no newsgroup selected';
146 # Ugh this can be silly expensive for big groups
147 $ng->mm->each_id_batch(sub {
149 more($self, join("\r\n", @$ary));
155 my ($date, $time, $gmt) = @_;
156 use Time::Local qw();
157 my ($YY, $MM, $DD) = unpack('A2A2A2', $date);
158 my ($hh, $mm, $ss) = unpack('A2A2A2', $time);
160 $gmt =~ /\A(?:UTC|GMT)\z/i or die "GM invalid: $gmt\n";
163 my @now = $gmt ? gmtime : localtime;
164 if ($YY > strftime('%y', @now)) {
165 my $cur_year = $now[5] + 1900;
166 $YY += int($cur_year / 1000) * 1000 - 100;
170 Time::Local::timegm($ss, $mm, $hh, $DD, $MM - 1, $YY);
172 Time::Local::timelocal($ss, $mm, $hh, $DD, $MM - 1, $YY);
177 my ($self, $ng) = @_;
178 my ($min, $max) = $ng->mm->minmax;
179 more($self, "$ng->{name} $max $min n") if defined $min && defined $max;
183 my ($self, $date, $time, $gmt, $dists) = @_;
184 my $ts = eval { parse_time($date, $time, $gmt) };
188 more($self, '231 list of new newsgroups follows');
189 foreach my $ng (values %{$self->{nntpd}->{groups}}) {
190 my $c = eval { $ng->mm->created_at } || 0;
191 next unless $c > $ts;
192 group_line($self, $ng);
198 return $_[0] = qr/.*/ if (!defined $_[0] || $_[0] eq '*');
201 use Digest::SHA qw(sha1_hex);
204 $tmp =~ s#(?<!\\)\[(.+)(?<!\\)\]#
206 my $key = sha1_hex($orig . $salt);
207 $orig =~ s/([^\w\-])+/\Q$1/g;
211 my %map = ('*' => '.*', '?' => '.' );
212 $tmp =~ s#(?<!\\)([^\w\\])#$map{$1} || "\Q$1"#ge;
214 $tmp =~ s#([a-f0-9]{40})#
215 my $orig = $keep{$1};
216 defined $orig ? $orig : $1;
219 $_[0] = qr/\A$tmp\z/;
223 return $_[0] = qr/\A\z/ unless defined $_[0];
224 my %map = ('*' => '.*', ',' => '|');
225 $_[0] =~ s!(.)!$map{$1} || "\Q$1"!ge;
226 $_[0] = qr/\A(?:$_[0])\z/;
230 my ($self, $newsgroups, $date, $time, $gmt, $dists) = @_;
231 my $ts = eval { parse_time($date, $time, $gmt) };
233 more($self, '230 list of new articles by message-id follows');
234 my ($keep, $skip) = split('!', $newsgroups, 2);
238 foreach my $ng (values %{$self->{nntpd}->{groups}}) {
239 $ng->{name} =~ $keep or next;
240 $ng->{name} =~ $skip and next;
241 my $srch = $ng->search or next;
244 return '.' unless @srch;
247 my $opts = { asc => 1, limit => 1000, offset => 0 };
249 my $end = 0xffffffff; # would like to read 4 billion messages?
250 $self->long_response(0, $end, sub {
253 my $res = $srch->query($ts, $opts);
254 my $msgs = $res->{msgs};
255 if (my $nr = scalar @$msgs) {
257 join(">\r\n<", map { $_->mid } @$msgs ).
259 $opts->{offset} += $nr;
262 if (@srch) { # continue onto next newsgroup
264 } else { # break out of the long response.
272 my ($self, $group) = @_;
273 my $no_such = '411 no such news group';
274 my $ng = $self->{nntpd}->{groups}->{$group} or return $no_such;
277 my ($min, $max) = $ng->mm->minmax;
280 $self->{article} = $min;
281 my $est_size = $max - $min;
282 "211 $est_size $min $max $group";
286 my ($self, $off) = @_;
287 my $ng = $self->{ng} or return '412 no newsgroup selected';
289 my $n = $self->{article};
290 defined $n or return '420 no current article has been selected';
293 my $mid = $ng->mm->mid_for($n);
295 $n = $off > 0 ? 'next' : 'previous';
296 return "421 no $n article in this group";
298 $self->{article} = $n;
299 "223 $n <$mid> article retrieved - request text separately";
302 sub cmd_next { article_adj($_[0], 1) }
303 sub cmd_last { article_adj($_[0], -1) }
305 # We want to encourage using email and CC-ing everybody involved to avoid
306 # the single-point-of-failure a single server provides.
309 my $ng = $self->{ng};
310 $ng ? "440 mailto:$ng->{address} to post" : '440 posting not allowed'
315 res($self, '205 closing connection - goodbye!');
321 my ($self, $art, $set_headers) = @_;
322 my $ng = $self->{ng} or return '412 no newsgroup has been selected';
326 if ($art =~ /\A\d+\z/o) {
327 $err = '423 no such article number in this group';
330 } elsif ($art =~ /\A<([^>]+)>\z/) {
331 $err = '430 no such article found';
333 $n = $ng->mm->num_for($mid);
334 defined $mid or return $err;
339 $err = '420 no current article has been selected';
340 $n = $self->{article};
341 defined $n or return $err;
343 $mid = $ng->mm->mid_for($n);
344 defined $mid or return $err;
347 my $o = 'HEAD:' . mid2path($mid);
348 my $s = eval { Email::Simple->new($ng->gcf->cat_file($o)) };
349 return $err unless $s;
351 $s->header_set('Newsgroups', $ng->{name});
352 $s->header_set('Lines', $s->body =~ tr!\n!\n!);
353 $s->header_set('Xref', "$ng->{domain} $ng->{name}:$n");
356 if ($set_headers == 2) {
357 $s->header_set('Bytes', bytes::length($s->as_string));
364 sub simple_body_write {
368 $body =~ s/^\./../smg;
369 do_more($self, $body);
375 my $h = $s->header_obj;
376 $h->header_set('Bytes');
381 my ($self, $art) = @_;
382 my $r = $self->art_lookup($art, 1);
383 return $r unless ref $r;
384 my ($n, $mid, $s) = @$r;
385 more($self, "220 $n <$mid> article retrieved - head and body follow");
386 do_more($self, header_str($s));
387 do_more($self, "\r\n");
388 simple_body_write($self, $s);
392 my ($self, $art) = @_;
393 my $r = $self->art_lookup($art, 2);
394 return $r unless ref $r;
395 my ($n, $mid, $s) = @$r;
396 more($self, "221 $n <$mid> article retrieved - head follows");
397 do_more($self, header_str($s));
402 my ($self, $art) = @_;
403 my $r = $self->art_lookup($art, 0);
404 return $r unless ref $r;
405 my ($n, $mid, $s) = @$r;
406 more($self, "222 $n <$mid> article retrieved - body follows");
407 simple_body_write($self, $s);
411 my ($self, $art) = @_;
412 my $r = $self->art_lookup($art, 0);
413 return $r unless ref $r;
414 my ($n, $mid, undef) = @$r;
415 "223 $n <$mid> article retrieved - request text separately";
418 sub cmd_ihave { '435 article not wanted - do not send it' }
420 sub cmd_date { '111 '.strftime('%Y%m%d%H%M%S', gmtime(time)) }
424 more($self, '100 help text follows');
429 my ($self, $range) = @_;
430 my $ng = $self->{ng} or return '412 no news group has been selected';
431 defined $range or return '420 No article(s) selected';
433 my ($min, $max) = $ng->mm->minmax;
434 if ($range =~ /\A(\d+)\z/) {
436 } elsif ($range =~ /\A(\d+)-\z/) {
437 ($beg, $end) = ($1, $max);
438 } elsif ($range =~ /\A(\d+)-(\d+)\z/) {
439 ($beg, $end) = ($1, $2);
443 $beg = $min if ($beg < $min);
444 $end = $max if ($end > $max);
445 return '420 No article(s) selected' if ($beg > $end);
450 my ($r, $header) = @_;
451 $r = $r->[2]->header_obj->header($header);
452 defined $r or return;
453 $r =~ s/[\r\n\t]+/ /sg;
458 my ($self, $beg, $end, $cb) = @_;
459 die "BUG: nested long response" if $self->{long_res};
461 # make sure we disable reading during a long response,
462 # clients should not be sending us stuff and making us do more
463 # work while we are stream a response to them
464 $self->watch_read(0);
465 $self->{long_res} = sub {
466 # limit our own running time for fairness with other
467 # clients and to avoid buffering too much:
469 local $SIG{ALRM} = sub { $yield = 1 };
474 eval { $cb->(\$beg) };
475 } until (($err = $@) || $self->{closed} || $yield ||
476 $self->{write_buf_size} || ++$beg > $end);
479 if ($err || $self->{closed}) {
480 $self->{long_res} = undef;
481 warning("$err during long response") if $err;
482 $self->watch_read(1) unless $self->{closed};
483 } elsif ($yield || $self->{write_buf_size}) {
484 # no recursion, schedule another call ASAP
485 # but only after all pending writes are done
486 Danga::Socket->AddTimer(0, sub {
487 $self->write($self->{long_res});
490 $self->{long_res} = undef;
491 $self->watch_read(1);
495 $self->{long_res}->(); # kick off!
500 my ($self, $header, $range) = @_;
501 defined $self->{ng} or return '412 no news group currently selected';
502 unless (defined $range) {
503 defined($range = $self->{article}) or
504 return '420 no current article has been selected';
506 if ($range =~ /\A<(.+)>\z/) { # Message-ID
507 my $r = $self->art_lookup($range, 2);
508 return $r unless ref $r;
509 more($self, '221 Header follows');
510 if (defined($r = xhdr($r, $header))) {
511 more($self, "<$range> $r");
514 } else { # numeric range
515 my $r = get_range($self, $range);
516 return $r unless ref $r;
517 my ($beg, $end) = @$r;
518 more($self, '221 Header follows');
519 $self->long_response($beg, $end, sub {
521 $r = $self->art_lookup($$i, 2);
522 return unless ref $r;
523 defined($r = xhdr($r, $header)) or return;
524 more($self, "$$i $r");
530 my ($self, $range) = @_;
531 my $r = get_range($self, $range);
532 return $r unless ref $r;
533 my ($beg, $end) = @$r;
534 more($self, "224 Overview information follows for $beg to $end");
535 $self->long_response($beg, $end, sub {
537 my $r = $self->art_lookup($$i, 2);
538 return unless ref $r;
539 more($self, join("\t", $r->[0],
541 my $h = xhdr($r, $_);
542 defined $h ? $h : '';
548 my ($self, $line) = @_;
549 do_write($self, $line . "\r\n");
553 my ($self, $line) = @_;
554 do_more($self, $line . "\r\n");
558 my ($self, $data) = @_;
559 my $done = $self->write($data);
560 die if $self->{closed};
562 # Do not watch for readability if we have data in the queue,
563 # instead re-enable watching for readability when we can
564 $self->watch_read(0) if (!$done || $self->{long_res});
569 use constant MSG_MORE => ($^O eq 'linux') ? 0x8000 : 0;
572 my ($self, $data) = @_;
573 if (MSG_MORE && !$self->{write_buf_size}) {
574 my $n = send($self->{sock}, $data, MSG_MORE);
576 my $dlen = bytes::length($data);
577 return 1 if $n == $dlen; # all done!
578 $data = bytes::substr($data, $n, $dlen - $n);
581 $self->do_write($data);
584 # callbacks for by Danga::Socket
586 sub event_hup { $_[0]->close }
587 sub event_err { $_[0]->close }
591 # only continue watching for readability when we are done writing:
592 $self->write(undef) == 1 and $self->watch_read(1);
597 use constant LINE_MAX => 512; # RFC 977 section 2.3
599 my $buf = $self->read(LINE_MAX) or return $self->close;
600 while ($r > 0 && $$buf =~ s/\A([^\r\n]+)\r?\n//) {
602 my $t0 = [ gettimeofday ];
603 $r = eval { $self->process_line($line) };
604 printf(STDERR "$line %0.6f\n",
605 tv_interval($t0, [gettimeofday]));
607 return $self->close if $r < 0;
608 my $len = bytes::length($$buf);
609 return $self->close if ($len >= LINE_MAX);
610 $self->push_back_read($buf) if ($len);
613 sub warning { print STDERR @_, "\n" }