X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FGzipFilter.pm;h=d72ad3c88da55779e9f2c953a63f61040a90c123;hb=5afb24f06627641e3fed608e807b5ab628cda348;hp=0a6c56a5da9beb27fa0c88356e379c42fb35853d;hpb=f982abbaa39e07090d3fb641fd9de25434c0d74c;p=public-inbox.git
diff --git a/lib/PublicInbox/GzipFilter.pm b/lib/PublicInbox/GzipFilter.pm
index 0a6c56a5..d72ad3c8 100644
--- a/lib/PublicInbox/GzipFilter.pm
+++ b/lib/PublicInbox/GzipFilter.pm
@@ -1,35 +1,67 @@
# Copyright (C) 2020 all contributors
# License: AGPL-3.0+
-
-# Qspawn filter
+#
+# In public-inbox <=1.5.0, public-inbox-httpd favored "getline"
+# response bodies to take a "pull"-based approach to feeding
+# slow clients (as opposed to a more common "push" model).
+#
+# In newer versions, public-inbox-httpd supports a backpressure-aware
+# pull/push model which also accounts for slow git blob storage.
+# {async_next} callbacks only run when the DS {wbuf} is drained
+# {async_eml} callbacks only run when a blob arrives from git.
+#
+# We continue to support getline+close for generic PSGI servers.
package PublicInbox::GzipFilter;
use strict;
use parent qw(Exporter);
-use Compress::Raw::Zlib qw(Z_FINISH Z_OK);
+use Compress::Raw::Zlib qw(Z_OK);
+use PublicInbox::CompressNoop;
+use PublicInbox::Eml;
+use PublicInbox::GitAsyncCat;
+
our @EXPORT_OK = qw(gzf_maybe);
my %OPT = (-WindowBits => 15 + 16, -AppendOutput => 1);
my @GZIP_HDRS = qw(Vary Accept-Encoding Content-Encoding gzip);
-sub new { bless {}, shift }
+sub new { bless {}, shift } # qspawn filter
# for Qspawn if using $env->{'pi-httpd.async'}
sub attach {
- my ($self, $fh) = @_;
- $self->{fh} = $fh;
+ my ($self, $http_out) = @_;
+ $self->{http_out} = $http_out; # PublicInbox::HTTP::{Chunked,Identity}
$self
}
-# returns `0' and not `undef' on failure (see Www*Stream)
-sub gzf_maybe ($$) {
+sub gz_or_noop {
my ($res_hdr, $env) = @_;
- return 0 if (($env->{HTTP_ACCEPT_ENCODING}) // '') !~ /\bgzip\b/;
- my ($gz, $err) = Compress::Raw::Zlib::Deflate->new(%OPT);
- return 0 if $err != Z_OK;
+ if (($env->{HTTP_ACCEPT_ENCODING} // '') =~ /\bgzip\b/) {
+ $env->{'plack.skip-deflater'} = 1;
+ push @$res_hdr, @GZIP_HDRS;
+ gzip_or_die();
+ } else {
+ PublicInbox::CompressNoop::new();
+ }
+}
- # in case Plack::Middleware::Deflater is loaded:
- $env->{'plack.skip-deflater'} = 1;
- push @$res_hdr, @GZIP_HDRS;
- bless { gz => $gz }, __PACKAGE__;
+sub gzf_maybe ($$) { bless { gz => gz_or_noop(@_) }, __PACKAGE__ }
+
+sub psgi_response {
+ my ($self, $code, $res_hdr, $next_cb, $eml_cb) = @_;
+ my $env = $self->{env};
+ $self->{gz} //= gz_or_noop($res_hdr, $env);
+ if ($env->{'pi-httpd.async'}) {
+ $self->{async_next} = $next_cb;
+ $self->{async_eml} = $eml_cb;
+ my $http = $env->{'psgix.io'}; # PublicInbox::HTTP
+ $http->{forward} = $self;
+ sub {
+ my ($wcb) = @_; # -httpd provided write callback
+ $self->{http_out} = $wcb->([$code, $res_hdr]);
+ $next_cb->($http); # start stepping
+ };
+ } else { # generic PSGI code path
+ [ $code, $res_hdr, $self ];
+ }
}
sub qsp_maybe ($$) {
@@ -72,7 +104,7 @@ sub translate ($$) {
$self->{zbuf} = $zbuf;
'';
} else { # undef == EOF
- my $err = $gz->flush($zbuf, Z_FINISH);
+ my $err = $gz->flush($zbuf);
die "gzip->flush: $err" if $err != Z_OK;
$zbuf;
}
@@ -80,7 +112,7 @@ sub translate ($$) {
sub write {
# my $ret = bytes::length($_[1]); # XXX does anybody care?
- $_[0]->{fh}->write(translate($_[0], $_[1]));
+ $_[0]->{http_out}->write(translate($_[0], $_[1]));
}
# similar to ->translate; use this when we're sure we know we have
@@ -102,16 +134,38 @@ sub zflush ($;$) {
$err = $gz->deflate($_[1], $zbuf);
die "gzip->deflate: $err" if $err != Z_OK;
}
- $err = $gz->flush($zbuf, Z_FINISH);
+ $err = $gz->flush($zbuf);
die "gzip->flush: $err" if $err != Z_OK;
$zbuf;
}
sub close {
my ($self) = @_;
- my $fh = delete $self->{fh};
- $fh->write(zflush($self));
- $fh->close;
+ if (my $http_out = delete $self->{http_out}) {
+ $http_out->write(zflush($self));
+ $http_out->close;
+ }
+}
+
+# this is public-inbox-httpd-specific
+sub async_blob_cb { # git->cat_async callback
+ my ($bref, $oid, $type, $size, $self) = @_;
+ my $http = $self->{env}->{'psgix.io'} or return; # client abort
+ my $smsg = $self->{smsg} or die 'BUG: no smsg';
+ if (!defined($oid)) {
+ # it's possible to have TOCTOU if an admin runs
+ # public-inbox-(edit|purge), just move onto the next message
+ return $http->next_step($self->{async_next});
+ }
+ $smsg->{blob} eq $oid or die "BUG: $smsg->{blob} != $oid";
+ $self->{async_eml}->($self, PublicInbox::Eml->new($bref));
+ $http->next_step($self->{async_next});
+}
+
+sub smsg_blob {
+ my ($self, $smsg) = @_;
+ git_async_cat($self->{-inbox}->git, $smsg->{blob},
+ \&async_blob_cb, $self);
}
1;