X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FGzipFilter.pm;h=e37f1f76bd4a886431224ed38aa8a3f838cc50b1;hb=7319f5d318a960eeb32a207d226eea7fd9ce2543;hp=6380f50e91f80ebd5e69d69b791cf36d5296e051;hpb=9f9290893bc241c5363d03b2d3c4acd15deb6d71;p=public-inbox.git diff --git a/lib/PublicInbox/GzipFilter.pm b/lib/PublicInbox/GzipFilter.pm index 6380f50e..e37f1f76 100644 --- a/lib/PublicInbox/GzipFilter.pm +++ b/lib/PublicInbox/GzipFilter.pm @@ -1,7 +1,16 @@ -# Copyright (C) 2020 all contributors +# Copyright (C) 2020-2021 all contributors # License: AGPL-3.0+ - -# Qspawn filter +# +# In public-inbox <=1.5.0, public-inbox-httpd favored "getline" +# response bodies to take a "pull"-based approach to feeding +# slow clients (as opposed to a more common "push" model). +# +# In newer versions, public-inbox-httpd supports a backpressure-aware +# pull/push model which also accounts for slow git blob storage. +# async_next callbacks only run when the DS {wbuf} is drained +# async_eml callbacks only run when a blob arrives from git. +# +# We continue to support getline+close for generic PSGI servers. package PublicInbox::GzipFilter; use strict; use parent qw(Exporter); @@ -14,12 +23,12 @@ our @EXPORT_OK = qw(gzf_maybe); my %OPT = (-WindowBits => 15 + 16, -AppendOutput => 1); my @GZIP_HDRS = qw(Vary Accept-Encoding Content-Encoding gzip); -sub new { bless {}, shift } +sub new { bless {}, shift } # qspawn filter # for Qspawn if using $env->{'pi-httpd.async'} sub attach { my ($self, $http_out) = @_; - $self->{http_out} = $http_out; + $self->{http_out} = $http_out; # PublicInbox::HTTP::{Chunked,Identity} $self } @@ -37,20 +46,20 @@ sub gz_or_noop { sub gzf_maybe ($$) { bless { gz => gz_or_noop(@_) }, __PACKAGE__ } sub psgi_response { - my ($self, $code, $res_hdr, $next_cb, $eml_cb) = @_; - my $env = $self->{env}; - $self->{gz} //= gz_or_noop($res_hdr, $env); - if ($env->{'pi-httpd.async'}) { - $self->{async_next} = $next_cb; - $self->{async_eml} = $eml_cb; - my $http = $env->{'psgix.io'}; # PublicInbox::HTTP + # $code may be an HTTP response code (e.g. 200) or a CODE ref (mbox_hdr) + my ($self, $code, $res_hdr) = @_; + if ($self->{env}->{'pi-httpd.async'}) { + my $http = $self->{env}->{'psgix.io'}; # PublicInbox::HTTP $http->{forward} = $self; sub { my ($wcb) = @_; # -httpd provided write callback - $self->{http_out} = $wcb->([$code, $res_hdr]); - $next_cb->($http); # start stepping + $self->{wcb_args} = [ $code, $res_hdr, $wcb ]; + $self->can('async_next')->($http); # start stepping }; } else { # generic PSGI code path + ref($code) eq 'CODE' and + ($code, $res_hdr) = @{$code->($self)}; + $self->{gz} //= gz_or_noop($res_hdr, $self->{env}); [ $code, $res_hdr, $self ]; } } @@ -77,6 +86,12 @@ sub gzip_or_die () { $gz; } +sub gone { # what: search/over/mm + my ($ctx, $what) = @_; + warn "W: `$ctx->{ibx}->{name}' $what went away unexpectedly\n"; + undef; +} + # for GetlineBody (via Qspawn) when NOT using $env->{'pi-httpd.async'} # Also used for ->getline callbacks sub translate ($$) { @@ -101,15 +116,28 @@ sub translate ($$) { } } +sub http_out ($) { + my ($self) = @_; + $self->{http_out} // do { + my $args = delete $self->{wcb_args} // return undef; + my $wcb = pop @$args; # from PublicInbox:HTTP async + # $args->[0] may be \&mbox_hdr or similar + $args = $args->[0]->($self) if ref($args->[0]) eq 'CODE'; + $self->{gz} //= gz_or_noop($args->[1], $self->{env}); + $self->{http_out} = $wcb->($args); # $wcb->([$code, $hdr_ary]) + }; +} + sub write { # my $ret = bytes::length($_[1]); # XXX does anybody care? - $_[0]->{http_out}->write(translate($_[0], $_[1])); + http_out($_[0])->write(translate($_[0], $_[1])); } # similar to ->translate; use this when we're sure we know we have # more data to buffer after this sub zmore { my $self = $_[0]; # $_[1] => input + http_out($self); my $err = $self->{gz}->deflate($_[1], $self->{zbuf}); die "gzip->deflate: $err" if $err != Z_OK; undef; @@ -132,31 +160,51 @@ sub zflush ($;$) { sub close { my ($self) = @_; - if (my $http_out = delete $self->{http_out}) { - $http_out->write(zflush($self)); - $http_out->close; + my $http_out = http_out($self) // return; + $http_out->write(zflush($self)); + delete($self->{http_out})->close; +} + +sub bail { + my $self = shift; + if (my $env = $self->{env}) { + warn @_, "\n"; + my $http = $env->{'psgix.io'} or return; # client abort + eval { $http->close }; # should hit our close + warn "E: error in http->close: $@" if $@; + eval { $self->close }; # just in case... + warn "E: error in self->close: $@" if $@; + } else { + warn @_, "\n"; } } # this is public-inbox-httpd-specific sub async_blob_cb { # git->cat_async callback my ($bref, $oid, $type, $size, $self) = @_; - my $http = $self->{env}->{'psgix.io'} or return; # client abort - my $smsg = $self->{smsg} or die 'BUG: no smsg'; + my $http = $self->{env}->{'psgix.io'}; + $http->{forward} or return; # client aborted + my $smsg = $self->{smsg} or bail($self, 'BUG: no smsg'); if (!defined($oid)) { # it's possible to have TOCTOU if an admin runs # public-inbox-(edit|purge), just move onto the next message - return $http->next_step($self->{async_next}); + warn "E: $smsg->{blob} missing in $self->{ibx}->{inboxdir}\n"; + return $http->next_step($self->can('async_next')); + } + $smsg->{blob} eq $oid or bail($self, "BUG: $smsg->{blob} != $oid"); + eval { $self->async_eml(PublicInbox::Eml->new($bref)) }; + bail($self, "E: async_eml: $@") if $@; + if ($self->{-low_prio}) { + push(@{$self->{www}->{-low_prio_q}}, $self) == 1 and + PublicInbox::DS::requeue($self->{www}); + } else { + $http->next_step($self->can('async_next')); } - $smsg->{blob} eq $oid or die "BUG: $smsg->{blob} != $oid"; - $self->{async_eml}->($self, PublicInbox::Eml->new($bref)); - $http->next_step($self->{async_next}); } sub smsg_blob { my ($self, $smsg) = @_; - git_async_cat($self->{-inbox}->git, $smsg->{blob}, - \&async_blob_cb, $self); + ibx_async_cat($self->{ibx}, $smsg->{blob}, \&async_blob_cb, $self); } 1;