X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FGzipFilter.pm;h=20030433fa1e7ded0a95f3dc97ba27e0f8d5c828;hb=0821af5f21fdb083020ae2e3e79e4227ef59cd4f;hp=0a6c56a5da9beb27fa0c88356e379c42fb35853d;hpb=f982abbaa39e07090d3fb641fd9de25434c0d74c;p=public-inbox.git

diff --git a/lib/PublicInbox/GzipFilter.pm b/lib/PublicInbox/GzipFilter.pm
index 0a6c56a5..20030433 100644
--- a/lib/PublicInbox/GzipFilter.pm
+++ b/lib/PublicInbox/GzipFilter.pm
@@ -1,35 +1,65 @@
 # Copyright (C) 2020 all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-
-# Qspawn filter
+#
+# In public-inbox <=1.5.0, public-inbox-httpd favored "getline"
+# response bodies to take a "pull"-based approach to feeding
+# slow clients (as opposed to a more common "push" model).
+#
+# In newer versions, public-inbox-httpd supports a backpressure-aware
+# pull/push model which also accounts for slow git blob storage.
+# async_next callbacks only run when the DS {wbuf} is drained
+# async_eml callbacks only run when a blob arrives from git.
+#
+# We continue to support getline+close for generic PSGI servers.
 package PublicInbox::GzipFilter;
 use strict;
 use parent qw(Exporter);
-use Compress::Raw::Zlib qw(Z_FINISH Z_OK);
+use Compress::Raw::Zlib qw(Z_OK);
+use PublicInbox::CompressNoop;
+use PublicInbox::Eml;
+use PublicInbox::GitAsyncCat;
+
 our @EXPORT_OK = qw(gzf_maybe);
 my %OPT = (-WindowBits => 15 + 16, -AppendOutput => 1);
 my @GZIP_HDRS = qw(Vary Accept-Encoding Content-Encoding gzip);
 
-sub new { bless {}, shift }
+sub new { bless {}, shift } # qspawn filter
 
 # for Qspawn if using $env->{'pi-httpd.async'}
 sub attach {
-	my ($self, $fh) = @_;
-	$self->{fh} = $fh;
+	my ($self, $http_out) = @_;
+	$self->{http_out} = $http_out; # PublicInbox::HTTP::{Chunked,Identity}
 	$self
 }
 
-# returns `0' and not `undef' on failure (see Www*Stream)
-sub gzf_maybe ($$) {
+sub gz_or_noop {
 	my ($res_hdr, $env) = @_;
-	return 0 if (($env->{HTTP_ACCEPT_ENCODING}) // '') !~ /\bgzip\b/;
-	my ($gz, $err) = Compress::Raw::Zlib::Deflate->new(%OPT);
-	return 0 if $err != Z_OK;
+	if (($env->{HTTP_ACCEPT_ENCODING} // '') =~ /\bgzip\b/) {
+		$env->{'plack.skip-deflater'} = 1;
+		push @$res_hdr, @GZIP_HDRS;
+		gzip_or_die();
+	} else {
+		PublicInbox::CompressNoop::new();
+	}
+}
 
-	# in case Plack::Middleware::Deflater is loaded:
-	$env->{'plack.skip-deflater'} = 1;
-	push @$res_hdr, @GZIP_HDRS;
-	bless { gz => $gz }, __PACKAGE__;
+sub gzf_maybe ($$) { bless { gz => gz_or_noop(@_) }, __PACKAGE__ }
+
+sub psgi_response {
+	my ($self, $code, $res_hdr) = @_;
+	my $env = $self->{env};
+	$self->{gz} //= gz_or_noop($res_hdr, $env);
+	if ($env->{'pi-httpd.async'}) {
+		my $http = $env->{'psgix.io'}; # PublicInbox::HTTP
+		$http->{forward} = $self;
+		sub {
+			my ($wcb) = @_; # -httpd provided write callback
+			$self->{http_out} = $wcb->([$code, $res_hdr]);
+			$self->can('async_next')->($http); # start stepping
+		};
+	} else { # generic PSGI code path
+		[ $code, $res_hdr, $self ];
+	}
 }
 
 sub qsp_maybe ($$) {
@@ -72,7 +102,7 @@ sub translate ($$) {
 		$self->{zbuf} = $zbuf;
 		'';
 	} else { # undef == EOF
-		my $err = $gz->flush($zbuf, Z_FINISH);
+		my $err = $gz->flush($zbuf);
 		die "gzip->flush: $err" if $err != Z_OK;
 		$zbuf;
 	}
@@ -80,7 +110,7 @@ sub translate ($$) {
 
 sub write {
 	# my $ret = bytes::length($_[1]); # XXX does anybody care?
-	$_[0]->{fh}->write(translate($_[0], $_[1]));
+	$_[0]->{http_out}->write(translate($_[0], $_[1]));
 }
 
 # similar to ->translate; use this when we're sure we know we have
@@ -102,16 +132,56 @@ sub zflush ($;$) {
 		$err = $gz->deflate($_[1], $zbuf);
 		die "gzip->deflate: $err" if $err != Z_OK;
 	}
-	$err = $gz->flush($zbuf, Z_FINISH);
+	$err = $gz->flush($zbuf);
 	die "gzip->flush: $err" if $err != Z_OK;
 	$zbuf;
 }
 
 sub close {
 	my ($self) = @_;
-	my $fh = delete $self->{fh};
-	$fh->write(zflush($self));
-	$fh->close;
+	if (my $http_out = delete $self->{http_out}) {
+		$http_out->write(zflush($self));
+		$http_out->close;
+	}
+}
+
+sub bail  {
+	my $self = shift;
+	if (my $env = $self->{env}) {
+		eval { $env->{'psgi.errors'}->print(@_, "\n") };
+		warn("E: error printing to psgi.errors: $@", @_) if $@;
+		my $http = $env->{'psgix.io'} or return; # client abort
+		eval { $http->close }; # should hit our close
+		warn "E: error in http->close: $@" if $@;
+		eval { $self->close }; # just in case...
+		warn "E: error in self->close: $@" if $@;
+	} else {
+		warn @_, "\n";
+	}
+}
+
+# this is public-inbox-httpd-specific
+sub async_blob_cb { # git->cat_async callback
+	my ($bref, $oid, $type, $size, $self) = @_;
+	my $http = $self->{env}->{'psgix.io'};
+	$http->{forward} or return; # client aborted
+	my $smsg = $self->{smsg} or bail($self, 'BUG: no smsg');
+	if (!defined($oid)) {
+		# it's possible to have TOCTOU if an admin runs
+		# public-inbox-(edit|purge), just move onto the next message
+		warn "E: $smsg->{blob} missing in $self->{-inbox}->{inboxdir}\n";
+		return $http->next_step($self->can('async_next'));
+	}
+	$smsg->{blob} eq $oid or bail($self, "BUG: $smsg->{blob} != $oid");
+	eval { $self->async_eml(PublicInbox::Eml->new($bref)) };
+	bail($self, "E: async_eml: $@") if $@;
+	$http->next_step($self->can('async_next'));
+}
+
+sub smsg_blob {
+	my ($self, $smsg) = @_;
+	git_async_cat($self->{-inbox}->git, $smsg->{blob},
+			\&async_blob_cb, $self);
 }
 
 1;