Virtually all of our responses are going to be gzipped, anyways.
This will allow us to utilize zlib as a buffering layer and
share common code for async blob retrieval responses.
To streamline this and allow GzipFilter to be a parent class,
we'll replace the NoopFilter with a similar CompressNoop class
which emulates the two Compress::Raw::Zlib::Deflate methods we
use.
This drops a bunch of redundant code and will hopefully make
upcoming WwwStream changes easier to reason about.
lib/PublicInbox/AdminEdit.pm
lib/PublicInbox/AltId.pm
lib/PublicInbox/Cgit.pm
+lib/PublicInbox/CompressNoop.pm
lib/PublicInbox/Config.pm
lib/PublicInbox/ContentHash.pm
lib/PublicInbox/DS.pm
lib/PublicInbox/NNTPD.pm
lib/PublicInbox/NNTPdeflate.pm
lib/PublicInbox/NewsWWW.pm
-lib/PublicInbox/NoopFilter.pm
lib/PublicInbox/Over.pm
lib/PublicInbox/OverIdx.pm
lib/PublicInbox/ParentPipe.pm
--- /dev/null
+# Copyright (C) 2020 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# Provide the same methods as Compress::Raw::Zlib::Deflate but
+# does no transformation of outgoing data
+package PublicInbox::CompressNoop;
+use strict;
+use Compress::Raw::Zlib qw(Z_OK);
+
+sub new { bless \(my $self), __PACKAGE__ }
+
+sub deflate { # ($self, $input, $output)
+ $_[2] .= $_[1];
+ Z_OK;
+}
+
+sub flush { # ($self, $output, $flags = Z_FINISH)
+ $_[1] //= '';
+ Z_OK;
+}
+
+1;
use strict;
use parent qw(Exporter);
use Compress::Raw::Zlib qw(Z_FINISH Z_OK);
+use PublicInbox::CompressNoop;
+use PublicInbox::Eml;
+use PublicInbox::GitAsyncCat;
+
our @EXPORT_OK = qw(gzf_maybe);
my %OPT = (-WindowBits => 15 + 16, -AppendOutput => 1);
my @GZIP_HDRS = qw(Vary Accept-Encoding Content-Encoding gzip);
# for Qspawn if using $env->{'pi-httpd.async'}
sub attach {
- my ($self, $fh) = @_;
- $self->{fh} = $fh;
+ my ($self, $http_out) = @_;
+ $self->{http_out} = $http_out;
$self
}
-# returns `0' and not `undef' on failure (see Www*Stream)
-sub gzf_maybe ($$) {
+sub gz_or_noop {
my ($res_hdr, $env) = @_;
- return 0 if (($env->{HTTP_ACCEPT_ENCODING}) // '') !~ /\bgzip\b/;
- my ($gz, $err) = Compress::Raw::Zlib::Deflate->new(%OPT);
- return 0 if $err != Z_OK;
+ if (($env->{HTTP_ACCEPT_ENCODING} // '') =~ /\bgzip\b/) {
+ $env->{'plack.skip-deflater'} = 1;
+ push @$res_hdr, @GZIP_HDRS;
+ gzip_or_die();
+ } else {
+ PublicInbox::CompressNoop::new();
+ }
+}
- # in case Plack::Middleware::Deflater is loaded:
- $env->{'plack.skip-deflater'} = 1;
- push @$res_hdr, @GZIP_HDRS;
- bless { gz => $gz }, __PACKAGE__;
+sub gzf_maybe ($$) { bless { gz => gz_or_noop(@_) }, __PACKAGE__ }
+
+sub psgi_response {
+ my ($self, $code, $res_hdr, $next_cb, $eml_cb) = @_;
+ my $env = $self->{env};
+ $self->{gz} //= gz_or_noop($res_hdr, $env);
+ if ($env->{'pi-httpd.async'}) {
+ $self->{async_next} = $next_cb;
+ $self->{async_eml} = $eml_cb;
+ my $http = $env->{'psgix.io'}; # PublicInbox::HTTP
+ $http->{forward} = $self;
+ sub {
+ my ($wcb) = @_; # -httpd provided write callback
+ $self->{http_out} = $wcb->([$code, $res_hdr]);
+ $next_cb->($http); # start stepping
+ };
+ } else { # generic PSGI code path
+ [ $code, $res_hdr, $self ];
+ }
}
sub qsp_maybe ($$) {
sub write {
# my $ret = bytes::length($_[1]); # XXX does anybody care?
- $_[0]->{fh}->write(translate($_[0], $_[1]));
+ $_[0]->{http_out}->write(translate($_[0], $_[1]));
}
# similar to ->translate; use this when we're sure we know we have
sub close {
my ($self) = @_;
- my $fh = delete $self->{fh};
- $fh->write(zflush($self));
- $fh->close;
+ if (my $http_out = delete $self->{http_out}) {
+ $http_out->write(zflush($self));
+ $http_out->close;
+ }
+}
+
+# this is public-inbox-httpd-specific
+sub async_blob_cb { # git->cat_async callback
+ my ($bref, $oid, $type, $size, $self) = @_;
+ my $http = $self->{env}->{'psgix.io'} or return; # client abort
+ my $smsg = $self->{smsg} or die 'BUG: no smsg';
+ if (!defined($oid)) {
+ # it's possible to have TOCTOU if an admin runs
+ # public-inbox-(edit|purge), just move onto the next message
+ return $http->next_step($self->{async_next});
+ }
+ $smsg->{blob} eq $oid or die "BUG: $smsg->{blob} != $oid";
+ $self->{async_eml}->($self, PublicInbox::Eml->new($bref));
+ $http->next_step($self->{async_next});
+}
+
+sub smsg_blob {
+ my ($self, $smsg) = @_;
+ git_async_cat($self->{-inbox}->git, $smsg->{blob},
+ \&async_blob_cb, $self);
}
1;
# more common "push" model)
package PublicInbox::Mbox;
use strict;
-use warnings;
+use parent 'PublicInbox::GzipFilter';
use PublicInbox::MID qw/mid_escape/;
use PublicInbox::Hval qw/to_filename/;
use PublicInbox::Smsg;
use PublicInbox::Eml;
-use PublicInbox::GitAsyncCat;
-use PublicInbox::GzipFilter qw(gzf_maybe);
# called by PSGI server as body response
# this gets called twice for every message, once to return the header,
my $smsg = $ctx->{smsg} or return;
my $ibx = $ctx->{-inbox};
my $eml = $ibx->smsg_eml($smsg) or return;
- $ctx->{smsg} = $ibx->over->next_by_mid($ctx->{mid}, @{$ctx->{id_prev}});
- msg_hdr($ctx, $eml, $smsg->{mid}) . msg_body($eml);
-}
-
-sub close { !!delete($_[0]->{http_out}) }
-
-sub mbox_async_step ($) { # public-inbox-httpd-only
- my ($ctx) = @_;
- if (my $smsg = $ctx->{smsg}) {
- git_async_cat($ctx->{-inbox}->git, $smsg->{blob},
- \&mbox_blob_cb, $ctx);
- } elsif (my $out = delete $ctx->{http_out}) {
- $out->close;
+ my $n = $ctx->{smsg} = $ibx->over->next_by_mid(@{$ctx->{next_arg}});
+ $ctx->zmore(msg_hdr($ctx, $eml, $smsg->{mid}));
+ if ($n) {
+ $ctx->translate(msg_body($eml));
+ } else { # last message
+ $ctx->zmore(msg_body($eml));
+ $ctx->zflush;
}
}
# called by PublicInbox::DS::write
-sub mbox_async_next {
+sub async_next {
my ($http) = @_; # PublicInbox::HTTP
my $ctx = $http->{forward} or return; # client aborted
eval {
- $ctx->{smsg} = $ctx->{-inbox}->over->next_by_mid(
- $ctx->{mid}, @{$ctx->{id_prev}});
- mbox_async_step($ctx);
+ my $smsg = $ctx->{smsg} or return $ctx->close;
+ $ctx->smsg_blob($smsg);
};
+ warn "E: $@" if $@;
}
-# this is public-inbox-httpd-specific
-sub mbox_blob_cb { # git->cat_async callback
- my ($bref, $oid, $type, $size, $ctx) = @_;
- my $http = $ctx->{env}->{'psgix.io'} or return; # client abort
- my $smsg = delete $ctx->{smsg} or die 'BUG: no smsg';
- if (!defined($oid)) {
- # it's possible to have TOCTOU if an admin runs
- # public-inbox-(edit|purge), just move onto the next message
- return $http->next_step(\&mbox_async_next);
- } else {
- $smsg->{blob} eq $oid or die "BUG: $smsg->{blob} != $oid";
- }
- my $eml = PublicInbox::Eml->new($bref);
- $ctx->{http_out}->write(msg_hdr($ctx, $eml, $smsg->{mid}));
- $ctx->{http_out}->write(msg_body($eml));
- $http->next_step(\&mbox_async_next);
+sub async_eml { # ->{async_eml} for async_blob_cb
+ my ($ctx, $eml) = @_;
+ my $smsg = delete $ctx->{smsg};
+ # next message
+ $ctx->{smsg} = $ctx->{-inbox}->over->next_by_mid(@{$ctx->{next_arg}});
+
+ $ctx->zmore(msg_hdr($ctx, $eml, $smsg->{mid}));
+ $ctx->{http_out}->write($ctx->translate(msg_body($eml)));
}
sub res_hdr ($$) {
[ msg_hdr($ctx, $eml, $ctx->{mid}) . msg_body($eml) ] ]
}
-sub stream_raw { # MboxGz response callback
- my ($ctx) = @_;
- delete($ctx->{smsg}) //
- $ctx->{-inbox}->over->next_by_mid($ctx->{mid},
- @{$ctx->{id_prev}});
-}
-
# /$INBOX/$MESSAGE_ID/raw
sub emit_raw {
my ($ctx) = @_;
- my $env = $ctx->{env};
- $ctx->{base_url} = $ctx->{-inbox}->base_url($env);
+ $ctx->{base_url} = $ctx->{-inbox}->base_url($ctx->{env});
my $over = $ctx->{-inbox}->over or return no_over_raw($ctx);
my ($id, $prev);
- my $smsg = $over->next_by_mid($ctx->{mid}, \$id, \$prev) or return;
- $ctx->{smsg} = $smsg;
+ my $mip = $ctx->{next_arg} = [ $ctx->{mid}, \$id, \$prev ];
+ my $smsg = $ctx->{smsg} = $over->next_by_mid(@$mip) or return;
my $res_hdr = res_hdr($ctx, $smsg->{subject});
- $ctx->{id_prev} = [ \$id, \$prev ];
-
- if (my $gzf = gzf_maybe($res_hdr, $env)) {
- $ctx->{gz} = delete $gzf->{gz};
- require PublicInbox::MboxGz;
- PublicInbox::MboxGz::response($ctx, \&stream_raw, $res_hdr);
- } elsif ($env->{'pi-httpd.async'}) {
- sub {
- my ($wcb) = @_; # -httpd provided write callback
- $ctx->{http_out} = $wcb->([200, $res_hdr]);
- $ctx->{env}->{'psgix.io'}->{forward} = $ctx;
- bless $ctx, __PACKAGE__;
- mbox_async_step($ctx); # start stepping
- };
- } else { # generic PSGI code path
- bless $ctx, __PACKAGE__; # respond to ->getline
- [ 200, $res_hdr, $ctx ];
- }
+ bless $ctx, __PACKAGE__;
+ $ctx->psgi_response(200, $res_hdr, \&async_next, \&async_eml);
}
sub msg_hdr ($$;$) {
use PublicInbox::Eml;
use PublicInbox::Hval qw/to_filename/;
use PublicInbox::Mbox;
-use PublicInbox::GitAsyncCat;
*msg_hdr = \&PublicInbox::Mbox::msg_hdr;
*msg_body = \&PublicInbox::Mbox::msg_body;
-# this is public-inbox-httpd-specific
-sub mboxgz_blob_cb { # git->cat_async callback
- my ($bref, $oid, $type, $size, $self) = @_;
- my $http = $self->{env}->{'psgix.io'} or return; # client abort
- my $smsg = delete $self->{smsg} or die 'BUG: no smsg';
- if (!defined($oid)) {
- # it's possible to have TOCTOU if an admin runs
- # public-inbox-(edit|purge), just move onto the next message
- return $http->next_step(\&mboxgz_async_next);
- } else {
- $smsg->{blob} eq $oid or die "BUG: $smsg->{blob} != $oid";
- }
- my $eml = PublicInbox::Eml->new($bref);
- $self->zmore(msg_hdr($self, $eml, $smsg->{mid}));
-
- # PublicInbox::HTTP::{Chunked,Identity}::write
- $self->{http_out}->write($self->translate(msg_body($eml)));
-
- $http->next_step(\&mboxgz_async_next);
-}
-
-# this is public-inbox-httpd-specific
-sub mboxgz_async_step ($) {
- my ($self) = @_;
- if (my $smsg = $self->{smsg} = $self->{cb}->($self)) {
- git_async_cat($self->{-inbox}->git, $smsg->{blob},
- \&mboxgz_blob_cb, $self);
- } elsif (my $out = delete $self->{http_out}) {
- $out->write($self->zflush);
- $out->close;
- }
-}
-
-# called by PublicInbox::DS::write
-sub mboxgz_async_next {
+sub async_next ($) {
my ($http) = @_; # PublicInbox::HTTP
- mboxgz_async_step($http->{forward});
-}
-
-# called by PublicInbox::HTTP::close, or any other PSGI server
-sub close { !!delete($_[0]->{http_out}) }
-
-sub response {
- my ($self, $cb, $res_hdr) = @_;
- $self->{cb} = $cb;
- bless $self, __PACKAGE__;
- if ($self->{env}->{'pi-httpd.async'}) {
- sub {
- my ($wcb) = @_; # -httpd provided write callback
- $self->{http_out} = $wcb->([200, $res_hdr]);
- $self->{env}->{'psgix.io'}->{forward} = $self;
- mboxgz_async_step($self); # start stepping
- };
- } else { # generic PSGI
- [ 200, $res_hdr, $self ];
- }
+ my $ctx = $http->{forward} or return;
+ eval {
+ $ctx->{smsg} = $ctx->{cb}->($ctx) or return $ctx->close;
+ $ctx->smsg_blob($ctx->{smsg});
+ };
+ warn "E: $@" if $@;
}
sub mbox_gz {
my ($self, $cb, $fn) = @_;
+ $self->{cb} = $cb;
$self->{base_url} = $self->{-inbox}->base_url($self->{env});
$self->{gz} = PublicInbox::GzipFilter::gzip_or_die();
$fn = to_filename($fn // 'no-subject');
$fn = 'no-subject' if $fn eq '';
# http://www.iana.org/assignments/media-types/application/gzip
- response($self, $cb, [ qw(Content-Type application/gzip),
- 'Content-Disposition', "inline; filename=$fn.mbox.gz" ]);
+ bless $self, __PACKAGE__;
+ my $res_hdr = [ 'Content-Type' => 'application/gzip',
+ 'Content-Disposition' => "inline; filename=$fn.mbox.gz" ];
+ $self->psgi_response(200, $res_hdr, \&async_next,
+ \&PublicInbox::Mbox::async_eml);
}
# called by Plack::Util::foreach or similar (generic PSGI)
+++ /dev/null
-# Copyright (C) 2020 all contributors <meta@public-inbox.org>
-# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-
-package PublicInbox::NoopFilter;
-use strict;
-
-sub new { bless \(my $self = ''), __PACKAGE__ }
-
-# noop workalike for PublicInbox::GzipFilter methods
-sub translate {
- my $self = $_[0];
- my $ret = $$self .= ($_[1] // '');
- $$self = '';
- $ret;
-}
-
-sub zmore {
- ${$_[0]} .= $_[1];
- undef;
-}
-
-sub zflush { translate($_[0], $_[1]) }
-
-1;
# more common "push" model)
package PublicInbox::WwwAtomStream;
use strict;
-use warnings;
+use parent 'PublicInbox::GzipFilter';
use POSIX qw(strftime);
use Digest::SHA qw(sha1_hex);
use PublicInbox::Address;
use PublicInbox::Hval qw(ascii_html mid_href);
use PublicInbox::MsgTime qw(msg_timestamp);
-use PublicInbox::GzipFilter qw(gzf_maybe);
-use PublicInbox::GitAsyncCat;
-
-# called by generic PSGI server after getline,
-# and also by PublicInbox::HTTP::close
-sub close { !!delete($_[0]->{http_out}) }
sub new {
my ($class, $ctx, $cb) = @_;
$ctx->{feed_base_url} = $ctx->{-inbox}->base_url($ctx->{env});
- $ctx->{cb} = $cb || \&close;
+ $ctx->{cb} = $cb || \&PublicInbox::GzipFilter::close;
$ctx->{emit_header} = 1;
bless $ctx, $class;
}
-# called by PublicInbox::DS::write
-sub atom_async_next {
+sub async_next ($) {
my ($http) = @_; # PublicInbox::HTTP
- atom_async_step($http->{forward});
-}
-
-# this is public-inbox-httpd-specific
-sub atom_blob_cb { # git->cat_async callback
- my ($bref, $oid, $type, $size, $ctx) = @_;
- my $http = $ctx->{env}->{'psgix.io'} or return; # client abort
- my $smsg = delete $ctx->{smsg} or die 'BUG: no smsg';
- if (!defined($oid)) {
- # it's possible to have TOCTOU if an admin runs
- # public-inbox-(edit|purge), just move onto the next message
- return $http->next_step(\&atom_async_next);
- } else {
- $smsg->{blob} eq $oid or die "BUG: $smsg->{blob} != $oid";
- }
- my $buf = feed_entry($ctx, $smsg, PublicInbox::Eml->new($bref));
- if (my $gzf = $ctx->{gzf}) {
- $buf = $gzf->translate($buf);
- }
- # PublicInbox::HTTP::{Chunked,Identity}::write
- $ctx->{http_out}->write($buf);
-
- $http->next_step(\&atom_async_next);
+ my $ctx = $http->{forward} or return;
+ eval {
+ if (my $smsg = $ctx->{smsg} = $ctx->{cb}->($ctx)) {
+ $ctx->smsg_blob($smsg);
+ } else {
+ $ctx->{http_out}->write($ctx->translate('</feed>'));
+ $ctx->close;
+ }
+ };
+ warn "E: $@" if $@;
}
-sub atom_async_step { # this is public-inbox-httpd-specific
- my ($ctx) = @_;
- if (my $smsg = $ctx->{smsg} = $ctx->{cb}->($ctx)) {
- git_async_cat($ctx->{-inbox}->git, $smsg->{blob},
- \&atom_blob_cb, $ctx);
- } elsif (my $out = delete $ctx->{http_out}) {
- if (my $gzf = delete $ctx->{gzf}) {
- $out->write($gzf->zflush);
- }
- $out->close;
- }
+sub async_eml { # ->{async_eml} for async_blob_cb
+ my ($ctx, $eml) = @_;
+ my $smsg = delete $ctx->{smsg};
+ $ctx->{http_out}->write($ctx->translate(feed_entry($ctx, $smsg, $eml)))
}
sub response {
my ($class, $ctx, $code, $cb) = @_;
my $res_hdr = [ 'Content-Type' => 'application/atom+xml' ];
$class->new($ctx, $cb);
- $ctx->{gzf} = gzf_maybe($res_hdr, $ctx->{env});
- if ($ctx->{env}->{'pi-httpd.async'}) {
- sub {
- my ($wcb) = @_; # -httpd provided write callback
- $ctx->{http_out} = $wcb->([200, $res_hdr]);
- $ctx->{env}->{'psgix.io'}->{forward} = $ctx;
- atom_async_step($ctx); # start stepping
- };
- } else {
- [ $code, $res_hdr, $ctx ];
- }
+ $ctx->psgi_response($code, $res_hdr, \&async_next, \&async_eml);
}
# called once for each message by PSGI server
sub getline {
my ($self) = @_;
- my $buf = do {
- if (my $middle = $self->{cb}) {
- if (my $smsg = $middle->($self)) {
- my $eml = $self->{-inbox}->smsg_eml($smsg) or
- return '';
- feed_entry($self, $smsg, $eml);
- } else {
- undef;
- }
- }
- } // (delete($self->{cb}) ? '</feed>' : undef);
-
- # gzf may be GzipFilter, `undef' or `0'
- my $gzf = $self->{gzf} or return $buf;
-
- return $gzf->translate($buf) if defined $buf;
- $self->{gzf} = 0; # next call to ->getline returns $buf (== undef)
- $gzf->translate(undef);
+ my $cb = $self->{cb} or return;
+ while (my $smsg = $cb->($self)) {
+ my $eml = $self->{-inbox}->smsg_eml($smsg) or next;
+ return $self->translate(feed_entry($self, $smsg, $eml));
+ }
+ delete $self->{cb};
+ $self->zflush('</feed>');
}
# private
use PublicInbox::Linkify;
use PublicInbox::View;
use PublicInbox::Inbox;
-use PublicInbox::NoopFilter;
use PublicInbox::GzipFilter qw(gzf_maybe);
use bytes (); # bytes::length
use HTTP::Date qw(time2str);
my ($env, $list) = @_;
my $h = [ 'Content-Type', 'text/html; charset=UTF-8',
'Content-Length', undef ];
- my $gzf = gzf_maybe($h, $env) || PublicInbox::NoopFilter::new();
+ my $gzf = gzf_maybe($h, $env);
$gzf->zmore('<html><head><title>' .
'public-inbox listing</title>' .
'</head><body><pre>');
use HTTP::Status qw(status_message);
use Errno qw(EACCES ENOTDIR ENOENT);
use URI::Escape qw(uri_escape_utf8);
-use PublicInbox::NoopFilter;
use PublicInbox::GzipFilter qw(gzf_maybe);
use PublicInbox::Hval qw(ascii_html);
use Plack::MIME;
my $path_info_html = ascii_html($path_info);
my $h = [qw(Content-Type text/html Content-Length), undef];
- my $gzf = gzf_maybe($h, $env) || PublicInbox::NoopFilter::new();
+ my $gzf = gzf_maybe($h, $env);
$gzf->zmore("<html><head><title>Index of $path_info_html</title>" .
${$self->{style}} .
"</head><body><pre>Index of $path_info_html</pre><hr><pre>\n");