X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FHTTP.pm;h=e65988bedf06fe2c5236e333bd7fc075021168b4;hb=c410bf707fb77b15698e14119dbf6e14e43479ba;hp=1346901ad52300fd4b9985c18ba912502ce0f649;hpb=55b707d788ce13696e4411389583e720ea6dab01;p=public-inbox.git diff --git a/lib/PublicInbox/HTTP.pm b/lib/PublicInbox/HTTP.pm index 1346901a..e65988be 100644 --- a/lib/PublicInbox/HTTP.pm +++ b/lib/PublicInbox/HTTP.pm @@ -1,4 +1,4 @@ -# Copyright (C) 2016-2019 all contributors +# Copyright (C) 2016-2021 all contributors # License: AGPL-3.0+ # # Generic PSGI server for convenience. It aims to provide @@ -6,19 +6,26 @@ # to learn different ways to admin both NNTP and HTTP components. # There's nothing which depends on public-inbox, here. # Each instance of this class represents a HTTP client socket - +# +# fields: +# httpd: PublicInbox::HTTPD ref +# env: PSGI env hashref +# input_left: bytes left to read in request body (e.g. POST/PUT) +# remote_addr: remote IP address as a string (e.g. "127.0.0.1") +# remote_port: peer port +# forward: response body object, response to ->getline + ->close +# alive: HTTP keepalive state: +# 0: drop connection when done +# 1: keep connection when done +# 2: keep connection, chunk responses package PublicInbox::HTTP; use strict; -use warnings; -use base qw(PublicInbox::DS); -use fields qw(httpd env input_left remote_addr remote_port forward alive); -use bytes (); # only for bytes::length +use parent qw(PublicInbox::DS); use Fcntl qw(:seek); use Plack::HTTPParser qw(parse_http_request); # XS or pure Perl use Plack::Util; use HTTP::Status qw(status_message); use HTTP::Date qw(time2str); -use IO::Handle; # ->write use PublicInbox::DS qw(msg_more); use PublicInbox::Syscall qw(EPOLLIN EPOLLONESHOT); use PublicInbox::Tmpfile; @@ -30,16 +37,6 @@ use constant { }; use Errno qw(EAGAIN); -my $pipelineq = []; -sub process_pipelineq () { - my $q = $pipelineq; - $pipelineq = []; - foreach (@$q) { - next unless $_->{sock}; - rbuf_process($_); - } -} - # Use the same configuration parameter as git since this is primarily # a slow-client sponge for git-http-backend # TODO: support per-respository http.maxRequestBuffer somehow... @@ -56,20 +53,18 @@ sub http_date () { sub new ($$$) { my ($class, $sock, $addr, $httpd) = @_; - my $self = fields::new($class); + my $self = bless { httpd => $httpd }, $class; my $ev = EPOLLIN; my $wbuf; - if (ref($sock) eq 'IO::Socket::SSL' && !$sock->accept_SSL) { + if ($sock->can('accept_SSL') && !$sock->accept_SSL) { return CORE::close($sock) if $! != EAGAIN; - $ev = PublicInbox::TLS::epollbit(); + $ev = PublicInbox::TLS::epollbit() or return CORE::close($sock); $wbuf = [ \&PublicInbox::DS::accept_tls_step ]; } - $self->SUPER::new($sock, $ev | EPOLLONESHOT); - $self->{httpd} = $httpd; $self->{wbuf} = $wbuf if $wbuf; ($self->{remote_addr}, $self->{remote_port}) = PublicInbox::Daemon::host_with_port($addr); - $self; + $self->SUPER::new($sock, $ev | EPOLLONESHOT); } sub event_step { # called by PublicInbox::DS @@ -80,51 +75,34 @@ sub event_step { # called by PublicInbox::DS # only read more requests if we've drained the write buffer, # otherwise we can be buffering infinitely w/o backpressure - return read_input($self) if defined $self->{env}; - my $rbuf = $self->{rbuf} // (\(my $x = '')); - $self->do_read($rbuf, 8192, bytes::length($$rbuf)) or return; - rbuf_process($self, $rbuf); -} - -sub rbuf_process { - my ($self, $rbuf) = @_; - $rbuf //= $self->{rbuf} // (\(my $x = '')); + return read_input($self) if ref($self->{env}); + my $rbuf = $self->{rbuf} // (\(my $x = '')); my %env = %{$self->{httpd}->{env}}; # full hash copy - my $r = parse_http_request($$rbuf, \%env); - - # We do not support Trailers in chunked requests, for now - # (they are rarely-used and git (as of 2.7.2) does not use them) - if ($r == -1 || $env{HTTP_TRAILER} || - # this length-check is necessary for PURE_PERL=1: - ($r == -2 && bytes::length($$rbuf) > 0x4000)) { - return quit($self, 400); - } - if ($r < 0) { # incomplete - $self->rbuf_idle($rbuf); - return $self->requeue; + my $r; + while (($r = parse_http_request($$rbuf, \%env)) < 0) { + # We do not support Trailers in chunked requests, for + # now (they are rarely-used and git (as of 2.7.2) does + # not use them). + # this length-check is necessary for PURE_PERL=1: + if ($r == -1 || $env{HTTP_TRAILER} || + ($r == -2 && length($$rbuf) > 0x4000)) { + return quit($self, 400); + } + $self->do_read($rbuf, 8192, length($$rbuf)) or return; } + return quit($self, 400) if grep(/\s/, keys %env); # stop smugglers $$rbuf = substr($$rbuf, $r); - my $len = input_prepare($self, \%env); - defined $len or return write_err($self, undef); # EMFILE/ENFILE + my $len = input_prepare($self, \%env) // + return write_err($self, undef); # EMFILE/ENFILE $len ? read_input($self, $rbuf) : app_dispatch($self, undef, $rbuf); } -# IO::Handle::write returns boolean, this returns bytes written: -sub xwrite ($$$) { - my ($fh, $rbuf, $max) = @_; - my $w = bytes::length($$rbuf); - $w = $max if $w > $max; - $fh->write($$rbuf, $w) or return; - $w; -} - sub read_input ($;$) { my ($self, $rbuf) = @_; $rbuf //= $self->{rbuf} // (\(my $x = '')); my $env = $self->{env}; - return if $env->{REMOTE_ADDR}; # in app dispatch return read_input_chunked($self, $rbuf) if env_chunked($env); # env->{CONTENT_LENGTH} (identity) @@ -133,7 +111,7 @@ sub read_input ($;$) { while ($len > 0) { if ($$rbuf ne '') { - my $w = xwrite($input, $rbuf, $len); + my $w = syswrite($input, $$rbuf, $len); return write_err($self, $len) unless $w; $len -= $w; die "BUG: $len < 0 (w=$w)" if $len < 0; @@ -153,9 +131,10 @@ sub app_dispatch { my ($self, $input, $rbuf) = @_; $self->rbuf_idle($rbuf); my $env = $self->{env}; + $self->{env} = undef; # for exists() check in ->busy $env->{REMOTE_ADDR} = $self->{remote_addr}; $env->{REMOTE_PORT} = $self->{remote_port}; - if (my $host = $env->{HTTP_HOST}) { + if (defined(my $host = $env->{HTTP_HOST})) { $host =~ s/:([0-9]+)\z// and $env->{SERVER_PORT} = $1; $env->{SERVER_NAME} = $host; } @@ -174,7 +153,10 @@ sub app_dispatch { response_write($self, $env, $res); } }; - $self->close if $@; + if ($@) { + warn "response_write error: $@"; + $self->close; + } } sub response_header_write { @@ -226,7 +208,7 @@ sub response_header_write { sub chunked_write ($$) { my $self = $_[0]; return if $_[1] eq ''; - msg_more($self, sprintf("%x\r\n", bytes::length($_[1]))); + msg_more($self, sprintf("%x\r\n", length($_[1]))); msg_more($self, $_[1]); # use $self->write(\"\n\n") if you care about real-time @@ -239,22 +221,11 @@ sub identity_write ($$) { $self->write(\($_[1])) if $_[1] ne ''; } -sub next_request ($) { - my ($self) = @_; - if ($self->{rbuf}) { - # avoid recursion for pipelined requests - PublicInbox::DS::requeue(\&process_pipelineq) if !@$pipelineq; - push @$pipelineq, $self; - } else { # wait for next request - $self->requeue; - } -} - sub response_done { my ($self, $alive) = @_; delete $self->{env}; # we're no longer busy $self->write(\"0\r\n\r\n") if $alive == 2; - $self->write($alive ? \&next_request : \&close); + $self->write($alive ? $self->can('requeue') : \&close); } sub getline_pull { @@ -264,7 +235,7 @@ sub getline_pull { # limit our own running time for fairness with other # clients and to avoid buffering too much: my $buf = eval { - local $/ = \8192; + local $/ = \65536; $forward->getline; } if $forward; @@ -277,23 +248,23 @@ sub getline_pull { } if ($self->{sock}) { - my $wbuf = $self->{wbuf} //= []; - push @$wbuf, \&getline_pull; + # autovivify wbuf + my $new_size = push(@{$self->{wbuf}}, \&getline_pull); # wbuf may be populated by {chunked,identity}_write() # above, no need to rearm if so: - $self->requeue if scalar(@$wbuf) == 1; + $self->requeue if $new_size == 1; return; # likely } } elsif ($@) { - err($self, "response ->getline error: $@"); + warn "response ->getline error: $@"; $self->close; } # avoid recursion if (delete $self->{forward}) { eval { $forward->close }; if ($@) { - err($self, "response ->close error: $@"); + warn "response ->close error: $@"; $self->close; # idempotent } } @@ -324,48 +295,48 @@ sub response_write { } } -sub input_tmpfile ($) { - my $input = tmpfile('http.input', $_[0]->{sock}) or return; - $input->autoflush(1); - $input; -} - sub input_prepare { my ($self, $env) = @_; - my $input; - my $len = $env->{CONTENT_LENGTH}; - if ($len) { - if ($len > $MAX_REQUEST_BUFFER) { - quit($self, 413); - return; - } - $input = input_tmpfile($self); - } elsif (env_chunked($env)) { + my ($input, $len); + + # rfc 7230 3.3.2, 3.3.3,: favor Transfer-Encoding over Content-Length + my $hte = $env->{HTTP_TRANSFER_ENCODING}; + if (defined $hte) { + # rfc7230 3.3.3, point 3 says only chunked is accepted + # as the final encoding. Since neither public-inbox-httpd, + # git-http-backend, or our WWW-related code uses "gzip", + # "deflate" or "compress" as the Transfer-Encoding, we'll + # reject them: + return quit($self, 400) if $hte !~ /\Achunked\z/i; + $len = CHUNK_START; - $input = input_tmpfile($self); + $input = tmpfile('http.input', $self->{sock}); } else { - $input = $null_io; + $len = $env->{CONTENT_LENGTH}; + if (defined $len) { + # rfc7230 3.3.3.4 + return quit($self, 400) if $len !~ /\A[0-9]+\z/; + return quit($self, 413) if $len > $MAX_REQUEST_BUFFER; + $input = $len ? tmpfile('http.input', $self->{sock}) + : $null_io; + } else { + $input = $null_io; + } } # TODO: expire idle clients on ENFILE / EMFILE - return unless $input; - - $env->{'psgi.input'} = $input; + $env->{'psgi.input'} = $input // return; $self->{env} = $env; $self->{input_left} = $len || 0; } -sub env_chunked { ($_[0]->{HTTP_TRANSFER_ENCODING} || '') =~ /\bchunked\b/i } - -sub err ($$) { - eval { $_[0]->{httpd}->{env}->{'psgi.errors'}->print($_[1]."\n") }; -} +sub env_chunked { ($_[0]->{HTTP_TRANSFER_ENCODING} // '') =~ /\Achunked\z/i } sub write_err { my ($self, $len) = @_; my $msg = $! || '(zero write)'; $msg .= " ($len bytes remaining)" if defined $len; - err($self, "error buffering to input: $msg"); + warn "error buffering to input: $msg"; quit($self, 500); } @@ -374,7 +345,7 @@ sub recv_err { if ($! == EAGAIN) { # epoll/kevent watch already set by do_read $self->{input_left} = $len; } else { - err($self, "error reading input: $! ($len bytes remaining)"); + warn "error reading input: $! ($len bytes remaining)"; } } @@ -389,12 +360,12 @@ sub read_input_chunked { # unlikely... $$rbuf =~ s/\A\r\n//s and return app_dispatch($self, $input, $rbuf); - return quit($self, 400) if bytes::length($$rbuf) > 2; + return quit($self, 400) if length($$rbuf) > 2; } if ($len == CHUNK_END) { if ($$rbuf =~ s/\A\r\n//s) { $len = CHUNK_START; - } elsif (bytes::length($$rbuf) > 2) { + } elsif (length($$rbuf) > 2) { return quit($self, 400); } } @@ -404,14 +375,14 @@ sub read_input_chunked { # unlikely... if (($len + -s $input) > $MAX_REQUEST_BUFFER) { return quit($self, 413); } - } elsif (bytes::length($$rbuf) > CHUNK_MAX_HDR) { + } elsif (length($$rbuf) > CHUNK_MAX_HDR) { return quit($self, 400); } # will break from loop since $len >= 0 } if ($len < 0) { # chunk header is trickled, read more - $self->do_read($rbuf, 8192, bytes::length($$rbuf)) or + $self->do_read($rbuf, 8192, length($$rbuf)) or return recv_err($self, $len); # (implicit) goto chunk_start if $r > 0; } @@ -420,7 +391,7 @@ sub read_input_chunked { # unlikely... # drain the current chunk until ($len <= 0) { if ($$rbuf ne '') { - my $w = xwrite($input, $rbuf, $len); + my $w = syswrite($input, $$rbuf, $len); return write_err($self, "$len chunk") if !$w; $len -= $w; if ($len == 0) { @@ -448,22 +419,28 @@ sub quit { my $h = "HTTP/1.1 $status " . status_message($status) . "\r\n\r\n"; $self->write(\$h); $self->close; + undef; # input_prepare expects this } sub close { my $self = $_[0]; - delete $self->{env}; # prevent circular references if (my $forward = delete $self->{forward}) { eval { $forward->close }; - err($self, "forward ->close error: $@") if $@; + warn "forward ->close error: $@" if $@; } $self->SUPER::close; # PublicInbox::DS::close } -# for graceful shutdown in PublicInbox::Daemon: -sub busy () { +sub busy { # for graceful shutdown in PublicInbox::Daemon: my ($self) = @_; - ($self->{rbuf} || $self->{env} || $self->{wbuf}); + defined($self->{rbuf}) || exists($self->{env}) || defined($self->{wbuf}) +} + +# runs $cb on the next iteration of the event loop at earliest +sub next_step { + my ($self, $cb) = @_; + return unless exists $self->{sock}; + $self->requeue if 1 == push(@{$self->{wbuf}}, $cb); } # Chunked and Identity packages are used for writing responses.