MANIFEST | 3 +-- lib/PublicInbox/DS.pm | 4 ++-- lib/PublicInbox/IMAP.pm | 10 +++++++--- lib/PublicInbox/IMAPD.pm | 2 +- lib/PublicInbox/IMAPdeflate.pm | 126 ----------------------------------------------------- lib/PublicInbox/NNTP.pm | 10 +++++++--- lib/PublicInbox/NNTPD.pm | 2 +- lib/PublicInbox/NNTPdeflate.pm => lib/PublicInbox/DSdeflate.pm | 13 +++++-------- xt/mem-imapd-tls.t | 4 ++-- diff --git a/MANIFEST b/MANIFEST index 923f5147482f5f2cb1a9acd36ddcfa5e6019f7f0..10547351aba27fb8dcc460a3d710364236f91de6 100644 --- a/MANIFEST +++ b/MANIFEST @@ -169,6 +169,7 @@ lib/PublicInbox/ContentHash.pm lib/PublicInbox/DS.pm lib/PublicInbox/DSKQXS.pm lib/PublicInbox/DSPoll.pm +lib/PublicInbox/DSdeflate.pm lib/PublicInbox/Daemon.pm lib/PublicInbox/DirIdle.pm lib/PublicInbox/DummyInbox.pm @@ -206,7 +207,6 @@ lib/PublicInbox/IMAP.pm lib/PublicInbox/IMAPClient.pm lib/PublicInbox/IMAPD.pm lib/PublicInbox/IMAPTracker.pm -lib/PublicInbox/IMAPdeflate.pm lib/PublicInbox/IMAPsearchqp.pm lib/PublicInbox/IPC.pm lib/PublicInbox/IdxStack.pm @@ -295,7 +295,6 @@ lib/PublicInbox/Msgmap.pm lib/PublicInbox/MultiGit.pm lib/PublicInbox/NNTP.pm lib/PublicInbox/NNTPD.pm -lib/PublicInbox/NNTPdeflate.pm lib/PublicInbox/NetNNTPSocks.pm lib/PublicInbox/NetReader.pm lib/PublicInbox/NetWriter.pm diff --git a/lib/PublicInbox/DS.pm b/lib/PublicInbox/DS.pm index fee31e3d132ab675ff11a9e16c9e623ddce8d1d3..ef483aacf2724a4bf73a50e64223c1b40374f3f2 100644 --- a/lib/PublicInbox/DS.pm +++ b/lib/PublicInbox/DS.pm @@ -648,8 +648,8 @@ $self->close; } } -sub zflush {} # overridden by NNTPdeflate and IMAPdeflate - +sub zflush {} # overridden by DSdeflate +sub compressed {} # overridden by DSdeflate sub long_response_done {} # overridden by Net::NNTP sub long_step { diff --git a/lib/PublicInbox/IMAP.pm b/lib/PublicInbox/IMAP.pm index ce0dce0f317e8fb40a2a0b9da160594cafeb4fcf..805f1102536d99d9cf77807f86e1b0b3e6290ce4 100644 --- a/lib/PublicInbox/IMAP.pm +++ b/lib/PublicInbox/IMAP.pm @@ -1212,8 +1212,6 @@ # to register it for socket-readiness notifications $self->requeue unless $pending; } -sub compressed { undef } - # RFC 4978 sub cmd_compress ($$$) { my ($self, $tag, $alg) = @_; @@ -1223,7 +1221,9 @@ # CRIME made TLS compression obsolete # return "$tag NO [COMPRESSIONACTIVE]\r\n" if $self->tls_compressed; - PublicInbox::IMAPdeflate->enable($self, $tag); + PublicInbox::IMAPdeflate->enable($self) or return + \"$tag BAD failed to activate compression\r\n"; + PublicInbox::DS::write($self, \"$tag OK DEFLATE active\r\n"); $self->requeue; undef } @@ -1268,5 +1268,9 @@ package PublicInbox::IMAP_preauth; our @ISA = qw(PublicInbox::IMAP); sub logged_in { 0 } + +package PublicInbox::IMAPdeflate; +use PublicInbox::DSdeflate; +our @ISA = qw(PublicInbox::DSdeflate PublicInbox::IMAP); 1; diff --git a/lib/PublicInbox/IMAPD.pm b/lib/PublicInbox/IMAPD.pm index d8814324f23e333ba4b5016f36f5c619b9d582b6..b24097a2453c10783aa334484ba3920d329d85b5 100644 --- a/lib/PublicInbox/IMAPD.pm +++ b/lib/PublicInbox/IMAPD.pm @@ -9,7 +9,7 @@ use v5.10.1; use PublicInbox::Config; use PublicInbox::ConfigIter; use PublicInbox::InboxIdle; -use PublicInbox::IMAPdeflate; # loads PublicInbox::IMAP +use PublicInbox::IMAP; use PublicInbox::DummyInbox; my $dummy = bless { uidvalidity => 0 }, 'PublicInbox::DummyInbox'; diff --git a/lib/PublicInbox/IMAPdeflate.pm b/lib/PublicInbox/IMAPdeflate.pm deleted file mode 100644 index d5929ef26ed5dfd85d2d74e62172b75758066bf0..0000000000000000000000000000000000000000 --- a/lib/PublicInbox/IMAPdeflate.pm +++ /dev/null @@ -1,126 +0,0 @@ -# Copyright (C) 2020-2021 all contributors -# License: AGPL-3.0+ -# TODO: reduce duplication from PublicInbox::NNTPdeflate - -# RFC 4978 -package PublicInbox::IMAPdeflate; -use strict; -use warnings; -use 5.010_001; -use base qw(PublicInbox::IMAP); -use Compress::Raw::Zlib; - -my %IN_OPT = ( - -Bufsize => 1024, - -WindowBits => -15, # RFC 1951 - -AppendOutput => 1, -); - -# global deflate context and buffer -my $zbuf = \(my $buf = ''); -my $zout; -{ - my $err; - ($zout, $err) = Compress::Raw::Zlib::Deflate->new( - # nnrpd (INN) and Compress::Raw::Zlib favor MemLevel=9, - # the zlib C library and git use MemLevel=8 as the default - # -MemLevel => 9, - -Bufsize => 65536, # same as nnrpd - -WindowBits => -15, # RFC 1951 - -AppendOutput => 1, - ); - $err == Z_OK or die "Failed to initialize zlib deflate stream: $err"; -} - -sub enable { - my ($class, $self, $tag) = @_; - my ($in, $err) = Compress::Raw::Zlib::Inflate->new(%IN_OPT); - if ($err != Z_OK) { - $self->err("Inflate->new failed: $err"); - $self->write(\"$tag BAD failed to activate compression\r\n"); - return; - } - $self->write(\"$tag OK DEFLATE active\r\n"); - bless $self, $class; - $self->{zin} = $in; -} - -# overrides PublicInbox::NNTP::compressed -sub compressed { 1 } - -sub do_read ($$$$) { - my ($self, $rbuf, $len, $off) = @_; - - my $zin = $self->{zin} or return; # closed - my $doff; - my $dbuf = delete($self->{dbuf}) // ''; - $doff = length($dbuf); - my $r = PublicInbox::DS::do_read($self, \$dbuf, $len, $doff) or return; - - # Workaround inflate bug appending to OOK scalars: - # - # We only have $off if the client is pipelining, and pipelining - # is where our substr() OOK optimization in event_step makes sense. - if ($off) { - my $copy = $$rbuf; - undef $$rbuf; - $$rbuf = $copy; - } - - # assert(length($$rbuf) == $off) as far as NNTP.pm is concerned - # -ConsumeInput is true, so $dbuf is automatically emptied - my $err = $zin->inflate($dbuf, $rbuf); - if ($err == Z_OK) { - $self->{dbuf} = $dbuf if $dbuf ne ''; - $r = length($$rbuf) and return $r; - # nothing ready, yet, get more, later - $self->requeue; - } else { - delete $self->{zin}; - $self->close; - } - 0; -} - -# override PublicInbox::DS::msg_more -sub msg_more ($$) { - my $self = $_[0]; - - # $_[1] may be a reference or not for ->deflate - my $err = $zout->deflate($_[1], $zbuf); - $err == Z_OK or die "->deflate failed $err"; - 1; -} - -sub zflush ($) { - my ($self) = @_; - - my $deflated = $zbuf; - $zbuf = \(my $next = ''); - - my $err = $zout->flush($deflated, Z_FULL_FLUSH); - $err == Z_OK or die "->flush failed $err"; - - # We can still let the lower socket layer do buffering: - PublicInbox::DS::msg_more($self, $$deflated); -} - -# compatible with PublicInbox::DS::write, so $_[1] may be a reference or not -sub write ($$) { - my $self = $_[0]; - return PublicInbox::DS::write($self, $_[1]) if ref($_[1]) eq 'CODE'; - - my $deflated = $zbuf; - $zbuf = \(my $next = ''); - - # $_[1] may be a reference or not for ->deflate - my $err = $zout->deflate($_[1], $deflated); - $err == Z_OK or die "->deflate failed $err"; - $err = $zout->flush($deflated, Z_FULL_FLUSH); - $err == Z_OK or die "->flush failed $err"; - - # We can still let the socket layer do buffering: - PublicInbox::DS::write($self, $deflated); -} - -1; diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm index 3929f81737901507e06c80994a694718617e3910..8ad7adc1fd0a3ca49a5635dc19c8b4c7c95b6c01 100644 --- a/lib/PublicInbox/NNTP.pm +++ b/lib/PublicInbox/NNTP.pm @@ -883,8 +883,6 @@ "224 Overview information follows for $$beg to $end\r\n"); $self->long_response(\&xover_i, @$r); } -sub compressed { undef } - sub cmd_starttls ($) { my ($self) = @_; my $sock = $self->{sock} or return; @@ -903,7 +901,9 @@ sub cmd_compress ($$) { my ($self, $alg) = @_; return "503 Only DEFLATE is supported\r\n" if uc($alg) ne 'DEFLATE'; return r502 if $self->compressed; - PublicInbox::NNTPdeflate->enable($self); + PublicInbox::NNTPdeflate->enable($self) or return + \"403 Unable to activate compression\r\n"; + PublicInbox::DS::write($self, \"206 Compression active\r\n"); $self->requeue; undef } @@ -984,5 +984,9 @@ sub busy { # for graceful shutdown in PublicInbox::Daemon: my ($self) = @_; defined($self->{rbuf}) || defined($self->{wbuf}) } + +package PublicInbox::NNTPdeflate; +use PublicInbox::DSdeflate; +our @ISA = qw(PublicInbox::DSdeflate PublicInbox::NNTP); 1; diff --git a/lib/PublicInbox/NNTPD.pm b/lib/PublicInbox/NNTPD.pm index 6e79f0be697fc909ac49ad301b54a4447eebfdc6..f31d43818e5570fa44f33a8c63fedc9610930956 100644 --- a/lib/PublicInbox/NNTPD.pm +++ b/lib/PublicInbox/NNTPD.pm @@ -9,7 +9,7 @@ use v5.10.1; use Sys::Hostname; use PublicInbox::Config; use PublicInbox::InboxIdle; -use PublicInbox::NNTPdeflate; # loads PublicInbox::NNTP +use PublicInbox::NNTP; sub new { my ($class) = @_; diff --git a/lib/PublicInbox/NNTPdeflate.pm b/lib/PublicInbox/DSdeflate.pm rename from lib/PublicInbox/NNTPdeflate.pm rename to lib/PublicInbox/DSdeflate.pm index 352d4842834816a0df1523575aa4a6ea7f3326b3..b5208e4368abd63626cf617eda1bd4ba4f00fb08 100644 --- a/lib/PublicInbox/NNTPdeflate.pm +++ b/lib/PublicInbox/DSdeflate.pm @@ -2,6 +2,7 @@ # Copyright (C) all contributors # License: AGPL-3.0+ # RFC 8054 NNTP COMPRESS DEFLATE implementation +# RFC 4978 IMAP COMPRESS=DEFLATE extension # # RSS usage for 10K idle-but-did-something NNTP clients on 64-bit: # TLS + DEFLATE[a] : 1.8 GB (MemLevel=9, 1.2 GB with MemLevel=8) @@ -14,14 +15,13 @@ # # [b] - memory-optimized implementation using a global deflate context. # It's less efficient in terms of compression, but way more # efficient in terms of server memory usage. -package PublicInbox::NNTPdeflate; +package PublicInbox::DSdeflate; use strict; -use 5.010_001; -use parent qw(PublicInbox::NNTP); +use v5.10.1; use Compress::Raw::Zlib; my %IN_OPT = ( - -Bufsize => PublicInbox::NNTP::LINE_MAX, + -Bufsize => 1024, -WindowBits => -15, # RFC 1951 -AppendOutput => 1, ); @@ -42,21 +42,18 @@ ); $err == Z_OK or die "Failed to initialize zlib deflate stream: $err"; } - sub enable { my ($class, $self) = @_; my ($in, $err) = Compress::Raw::Zlib::Inflate->new(%IN_OPT); if ($err != Z_OK) { $self->err("Inflate->new failed: $err"); - $self->write(\"403 Unable to activate compression\r\n"); return; } - $self->write(\"206 Compression active\r\n"); bless $self, $class; $self->{zin} = $in; } -# overrides PublicInbox::NNTP::compressed +# overrides PublicInbox::DS::compressed sub compressed { 1 } sub do_read ($$$$) { diff --git a/xt/mem-imapd-tls.t b/xt/mem-imapd-tls.t index 8992a6fc0d8dc96dfe835ad5d00bae179e0889be..d728ce32ac57b5aa34842f06936abc37cb2b597e 100644 --- a/xt/mem-imapd-tls.t +++ b/xt/mem-imapd-tls.t @@ -1,5 +1,5 @@ #!perl -w -# Copyright (C) 2020-2021 all contributors +# Copyright (C) all contributors # License: AGPL-3.0+ # Idle client memory usage test, particularly after EXAMINE when # Message Sequence Numbers are loaded @@ -221,7 +221,7 @@ package IMAPCdeflate; use strict; our @ISA; use Compress::Raw::Zlib; -use PublicInbox::IMAPdeflate; +use PublicInbox::IMAP; my %ZIN_OPT; BEGIN { @ISA = qw(IMAPC);