1 # Copyright (C) all contributors <meta@public-inbox.org>
2 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
4 # In public-inbox <=1.5.0, public-inbox-httpd favored "getline"
5 # response bodies to take a "pull"-based approach to feeding
6 # slow clients (as opposed to a more common "push" model).
8 # In newer versions, public-inbox-httpd supports a backpressure-aware
9 # pull/push model which also accounts for slow git blob storage.
10 # async_next callbacks only run when the DS {wbuf} is drained
11 # async_eml callbacks only run when a blob arrives from git.
13 # We continue to support getline+close for generic PSGI servers.
14 package PublicInbox::GzipFilter;
16 use parent qw(Exporter);
17 use Compress::Raw::Zlib qw(Z_OK);
18 use PublicInbox::CompressNoop;
20 use PublicInbox::GitAsyncCat;
22 our @EXPORT_OK = qw(gzf_maybe);
23 my %OPT = (-WindowBits => 15 + 16, -AppendOutput => 1);
24 my @GZIP_HDRS = qw(Vary Accept-Encoding Content-Encoding gzip);
26 sub new { bless {}, shift } # qspawn filter
28 # for Qspawn if using $env->{'pi-httpd.async'}
30 my ($self, $http_out) = @_;
31 $self->{http_out} = $http_out; # PublicInbox::HTTP::{Chunked,Identity}
36 my ($res_hdr, $env) = @_;
37 if (($env->{HTTP_ACCEPT_ENCODING} // '') =~ /\bgzip\b/) {
38 $env->{'plack.skip-deflater'} = 1;
39 push @$res_hdr, @GZIP_HDRS;
42 PublicInbox::CompressNoop::new();
46 sub gzf_maybe ($$) { bless { gz => gz_or_noop(@_) }, __PACKAGE__ }
49 # $code may be an HTTP response code (e.g. 200) or a CODE ref (mbox_hdr)
50 my ($self, $code, $res_hdr) = @_;
51 if ($self->{env}->{'pi-httpd.async'}) {
52 my $http = $self->{env}->{'psgix.io'}; # PublicInbox::HTTP
53 $http->{forward} = $self;
55 my ($wcb) = @_; # -httpd provided write callback
56 $self->{wcb_args} = [ $code, $res_hdr, $wcb ];
57 $self->can('async_next')->($http); # start stepping
59 } else { # generic PSGI code path
60 ref($code) eq 'CODE' and
61 ($code, $res_hdr) = @{$code->($self)};
62 $self->{gz} //= gz_or_noop($res_hdr, $self->{env});
63 [ $code, $res_hdr, $self ];
68 my ($res_hdr, $env) = @_;
69 return if ($env->{HTTP_ACCEPT_ENCODING} // '') !~ /\bgzip\b/;
70 my $hdr = join("\n", @$res_hdr);
71 return if $hdr !~ m!^Content-Type\n
72 (?:(?:text/(?:html|plain))|
73 application/atom\+xml)\b!ixsm;
74 return if $hdr =~ m!^Content-Encoding\ngzip\n!smi;
75 return if $hdr =~ m!^Content-Length\n[0-9]+\n!smi;
76 return if $hdr =~ m!^Transfer-Encoding\n!smi;
77 # in case Plack::Middleware::Deflater is loaded:
78 return if $env->{'plack.skip-deflater'}++;
79 push @$res_hdr, @GZIP_HDRS;
80 bless {}, __PACKAGE__;
84 my ($gz, $err) = Compress::Raw::Zlib::Deflate->new(%OPT);
85 $err == Z_OK or die "Deflate->new failed: $err";
89 sub gone { # what: search/over/mm
90 my ($ctx, $what) = @_;
91 warn "W: `$ctx->{ibx}->{name}' $what went away unexpectedly\n";
95 # for GetlineBody (via Qspawn) when NOT using $env->{'pi-httpd.async'}
96 # Also used for ->getline callbacks
98 my $self = shift; # $_[1] => input
100 # allocate the zlib context lazily here, instead of in ->new.
101 # Deflate contexts are memory-intensive and this object may
102 # be sitting in the Qspawn limiter queue for a while.
103 $self->{gz} //= gzip_or_die();
104 if (defined $_[0]) { # my $buf = $_[1];
106 length($self->{zbuf}) >= 8192 ? delete($self->{zbuf}) : '';
107 } else { # undef == EOF
112 # returns PublicInbox::HTTP::{Chunked,Identity}
115 $self->{http_out} // do {
116 my $args = delete $self->{wcb_args} // return undef;
117 my $wcb = pop @$args; # from PublicInbox:HTTP async
118 # $args->[0] may be \&mbox_hdr or similar
119 $args = $args->[0]->($self) if ref($args->[0]) eq 'CODE';
120 $self->{gz} //= gz_or_noop($args->[1], $self->{env});
121 $self->{http_out} = $wcb->($args); # $wcb->([$code, $hdr_ary])
126 # my $ret = bytes::length($_[1]); # XXX does anybody care?
127 http_out($_[0])->write(translate(@_));
132 open($_[0]->{zfh}, '>>', \($_[0]->{pbuf} //= '')) or
138 # similar to ->translate; use this when we're sure we know we have
139 # more data to buffer after this
142 my $zfh = delete $self->{zfh};
143 if (@_ > 1 || $zfh) {
144 print { $zfh // zfh($self) } @_;
145 @_ = (delete $self->{pbuf});
150 ($err = $self->{gz}->deflate($_[0], $self->{zbuf})) == Z_OK or
151 die "gzip->deflate: $err";
154 # flushes and returns the final bit of gzipped data
156 my $self = shift; # $_[1..Inf] => final input (optional)
157 zmore($self, @_) if scalar(@_) || $self->{zfh};
158 # not a bug, recursing on DS->write failure
159 my $gz = delete $self->{gz} // return '';
161 my $zbuf = delete $self->{zbuf};
162 ($err = $gz->flush($zbuf)) == Z_OK or die "gzip->flush: $err";
168 my $http_out = http_out($self) // return;
169 $http_out->write(zflush($self));
170 (delete($self->{http_out}) // return)->close;
175 if (my $env = $self->{env}) {
177 my $http = $env->{'psgix.io'} or return; # client abort
178 eval { $http->close }; # should hit our close
179 warn "E: error in http->close: $@" if $@;
180 eval { $self->close }; # just in case...
181 warn "E: error in self->close: $@" if $@;
187 # this is public-inbox-httpd-specific
188 sub async_blob_cb { # git->cat_async callback
189 my ($bref, $oid, $type, $size, $self) = @_;
190 my $http = $self->{env}->{'psgix.io'}; # PublicInbox::HTTP
191 $http->{forward} or return; # client aborted
192 my $smsg = $self->{smsg} or bail($self, 'BUG: no smsg');
193 if (!defined($oid)) {
194 # it's possible to have TOCTOU if an admin runs
195 # public-inbox-(edit|purge), just move onto the next message
196 warn "E: $smsg->{blob} missing in $self->{ibx}->{inboxdir}\n";
197 return $http->next_step($self->can('async_next'));
199 $smsg->{blob} eq $oid or bail($self, "BUG: $smsg->{blob} != $oid");
200 eval { $self->async_eml(PublicInbox::Eml->new($bref)) };
201 bail($self, "E: async_eml: $@") if $@;
202 if ($self->{-low_prio}) { # run via PublicInbox::WWW::event_step
203 push(@{$self->{www}->{-low_prio_q}}, $self) == 1 and
204 PublicInbox::DS::requeue($self->{www});
206 $http->next_step($self->can('async_next'));
211 my ($self, $smsg) = @_;
212 ibx_async_cat($self->{ibx}, $smsg->{blob}, \&async_blob_cb, $self);