www: switch to zadd for the majority of buffering

This allows us to focus string concatenations in one place to
allow Perl internal scratchpad optimizations to reuse memory.

Calling Compress::Raw::Zlib::deflate repeatedly proves too
expensive in terms of CPU cycles.

diff --git a/lib/PublicInbox/GzipFilter.pm b/lib/PublicInbox/GzipFilter.pm
index eb0046ce3f3a777916c4cb3cde33b7194799cf5c..1f11acb8095bee0b09864370a0274fef9d1d1728 100644 (file)
--- a/lib/PublicInbox/GzipFilter.pm
+++ b/lib/PublicInbox/GzipFilter.pm
@@ -127,15 +127,21 @@ sub write {
        http_out($_[0])->write(translate($_[0], $_[1]));
 }
 
+sub zadd {
+       my $self = shift;
+       $self->{pbuf} .= $_ for @_; # perl internal pad memory use here
+}
+
 # similar to ->translate; use this when we're sure we know we have
 # more data to buffer after this
 sub zmore {
        my $self = shift; # $_[1] => input
        http_out($self);
-       my $err;
+       my $x;
+       defined($x = delete($self->{pbuf})) and unshift(@_, $x);
        for (@_) {
-               $err = $self->{gz}->deflate($_, $self->{zbuf});
-               die "gzip->deflate: $err" if $err != Z_OK;
+               ($x = $self->{gz}->deflate($_, $self->{zbuf})) == Z_OK or
+                       die "gzip->deflate: $x";
        }
        undef;
 }
@@ -145,14 +151,14 @@ sub zflush ($;@) {
        my $self = shift; # $_[1..Inf] => final input (optional)
        my $zbuf = delete $self->{zbuf};
        my $gz = delete $self->{gz};
-       my $err;
+       my $x;
+       defined($x = delete($self->{pbuf})) and unshift(@_, $x);
        for (@_) { # it's a bug iff $gz is undef if @_ isn't empty, here:
-               $err = $gz->deflate($_, $zbuf);
-               die "gzip->deflate: $err" if $err != Z_OK;
+               ($x = $gz->deflate($_, $zbuf)) == Z_OK or
+                       die "gzip->deflate: $x";
        }
        $gz // return ''; # not a bug, recursing on DS->write failure
-       $err = $gz->flush($zbuf);
-       die "gzip->flush: $err" if $err != Z_OK;
+       ($x = $gz->flush($zbuf)) == Z_OK or die "gzip->flush: $x";
        $zbuf;
 }
 

diff --git a/lib/PublicInbox/Mbox.pm b/lib/PublicInbox/Mbox.pm
index 2ef8ff2ba48da7e52d2aebbde3d035c001c23503..cfe34d9c362843ded272074948ba4f995bf77e03 100644 (file)
--- a/lib/PublicInbox/Mbox.pm
+++ b/lib/PublicInbox/Mbox.pm
@@ -20,7 +20,7 @@ sub getline {
        my $ibx = $ctx->{ibx};
        my $eml = delete($ctx->{eml}) // $ibx->smsg_eml($smsg) // return;
        my $n = $ctx->{smsg} = $ibx->over->next_by_mid(@{$ctx->{next_arg}});
-       $ctx->zmore(msg_hdr($ctx, $eml));
+       $ctx->zadd(msg_hdr($ctx, $eml));
        if ($n) {
                $ctx->translate(msg_body($eml));
        } else { # last message

diff --git a/lib/PublicInbox/SearchView.pm b/lib/PublicInbox/SearchView.pm
index e0404e5fa5c724fb2a91eda9c60b171f8067682f..b18947ee1e94d3d32d8c2af0ce4de86bdcd953e7 100644 (file)
--- a/lib/PublicInbox/SearchView.pm
+++ b/lib/PublicInbox/SearchView.pm
@@ -331,7 +331,7 @@ sub mset_thread {
 # callback for PublicInbox::WwwStream::getline
 sub mset_thread_i {
        my ($ctx, $eml) = @_;
-       $ctx->zmore($ctx->html_top) if exists $ctx->{-html_tip};
+       $ctx->zadd($ctx->html_top) if exists $ctx->{-html_tip};
        $eml and return PublicInbox::View::eml_entry($ctx, $eml);
        my $smsg = shift @{$ctx->{msgs}} or
                $ctx->zmore(${delete($ctx->{skel})});

diff --git a/lib/PublicInbox/View.pm b/lib/PublicInbox/View.pm
index 630f1e42915e0635c4aa27adeac95765b7a02914..85dc3bd89d679d295ea0921e36d5acb69c515230 100644 (file)
--- a/lib/PublicInbox/View.pm
+++ b/lib/PublicInbox/View.pm
@@ -40,10 +40,10 @@ sub msg_page_i {
                                "../${\mid_href($smsg->{mid})}/" : '';
                if (_msg_page_prepare($eml, $ctx)) {
                        $eml->each_part(\&add_text_body, $ctx, 1);
-                       $ctx->zmore('</pre><hr>');
+                       $ctx->zadd('</pre><hr>');
                }
                html_footer($ctx, $ctx->{first_hdr}) if !$ctx->{smsg};
-               \''; # XXX TODO cleanup
+               ''; # XXX TODO cleanup
        } else { # called by WwwStream::async_next or getline
                $ctx->{smsg}; # may be undef
        }
@@ -58,7 +58,7 @@ sub no_over_html ($) {
        PublicInbox::WwwStream::init($ctx);
        if (_msg_page_prepare($eml, $ctx)) { # sets {-title_html}
                $eml->each_part(\&add_text_body, $ctx, 1);
-               $ctx->zmore('</pre><hr>');
+               $ctx->zadd('</pre><hr>');
        }
        html_footer($ctx, $eml);
        $ctx->html_done;
@@ -245,7 +245,7 @@ sub eml_entry {
        # scan through all parts, looking for displayable text
        $ctx->{mhref} = $mhref;
        $ctx->{changed_href} = "#e$id"; # for diffstat "files? changed,"
-       $ctx->zmore($rv); # XXX $rv is small, reuse below
+       $ctx->zadd($rv); # XXX $rv is small, reuse below
        $eml->each_part(\&add_text_body, $ctx, 1); # expensive
 
        # add the footer
@@ -386,7 +386,7 @@ sub pre_thread  { # walk_thread callback
 sub thread_eml_entry {
        my ($ctx, $eml) = @_;
        my ($beg, $end) = thread_adj_level($ctx, $ctx->{level});
-       $ctx->zmore($beg.'<pre>');
+       $ctx->zadd($beg.'<pre>');
        eml_entry($ctx, $eml) . '</pre>' . $end;
 }
 
@@ -414,15 +414,15 @@ sub stream_thread_i { # PublicInbox::WwwStream::getline callback
                                if (!$ghost_ok) { # first non-ghost
                                        $ctx->{-title_html} =
                                                ascii_html($smsg->{subject});
-                                       $ctx->zmore($ctx->html_top);
+                                       $ctx->zadd($ctx->html_top);
                                }
                                return $smsg;
                        }
                        # buffer the ghost entry and loop
-                       $ctx->zmore(ghost_index_entry($ctx, $lvl, $smsg));
+                       $ctx->zadd(ghost_index_entry($ctx, $lvl, $smsg));
                } else { # all done
-                       $ctx->zmore(join('', thread_adj_level($ctx, 0)));
-                       $ctx->zmore(${delete($ctx->{skel})});
+                       $ctx->zadd(join('', thread_adj_level($ctx, 0)));
+                       $ctx->zadd(${delete($ctx->{skel})});
                        return;
                }
        }
@@ -491,7 +491,7 @@ sub thread_html_i { # PublicInbox::WwwStream::getline callback
                my $smsg = $ctx->{smsg};
                if (exists $ctx->{-html_tip}) {
                        $ctx->{-title_html} = ascii_html($smsg->{subject});
-                       $ctx->zmore($ctx->html_top);
+                       $ctx->zadd($ctx->html_top);
                }
                return eml_entry($ctx, $eml);
        } else {
@@ -499,7 +499,7 @@ sub thread_html_i { # PublicInbox::WwwStream::getline callback
                        return $smsg if exists($smsg->{blob});
                }
                my $skel = delete($ctx->{skel}) or return; # all done
-               $ctx->zmore($$skel);
+               $ctx->zadd($$skel);
                undef;
        }
 }
@@ -560,7 +560,7 @@ sub add_text_body { # callback for each_part
        my $ct = $part->content_type || 'text/plain';
        my $fn = $part->filename;
        my ($s, $err) = msg_part_text($part, $ct);
-       $s // return $ctx->zmore(attach_link($ctx, $ct, $p, $fn) // '');
+       $s // return $ctx->zadd(attach_link($ctx, $ct, $p, $fn) // '');
        my $buf = $part->{is_submsg} ? submsg_hdr($ctx, $part)."\n" : '';
 
        # makes no difference to browsers, and don't screw up filename
@@ -612,18 +612,18 @@ sub add_text_body { # callback for each_part
                $buf .= attach_link($ctx, $ct, $p, $fn, $err) . "\n";
        }
        delete $part->{bdy}; # save memory
-       $ctx->zmore($buf);
+       $ctx->zadd($buf);
        undef $buf;
        for my $cur (@sections) { # $cur may be huge
                if ($cur =~ /\A>/) {
                        # we use a <span> here to allow users to specify
                        # their own color for quoted text
-                       $ctx->zmore(qq(<span\nclass="q">),
+                       $ctx->zadd(qq(<span\nclass="q">),
                                        $l->to_html($cur), '</span>');
                } elsif ($diff) {
                        flush_diff($ctx, \$cur);
                } else { # regular lines, OK
-                       $ctx->zmore($l->to_html($cur));
+                       $ctx->zadd($l->to_html($cur));
                }
                undef $cur; # free memory
        }
@@ -685,10 +685,10 @@ sub _msg_page_prepare {
                $hbuf .= qq[Message-ID: &lt;$x&gt; (<a href="raw">raw</a>)\n];
        }
        if (!$nr) { # first (and only) message, common case
-               $ctx->zmore($ctx->html_top, $hbuf);
+               $ctx->zadd($ctx->html_top, $hbuf);
        } else {
                delete $ctx->{-title_html};
-               $ctx->zmore($ctx->{-html_tip}, $hbuf);
+               $ctx->zadd($ctx->{-html_tip}, $hbuf);
        }
        $ctx->{-linkify} //= PublicInbox::Linkify->new;
        $hbuf = '';
@@ -699,7 +699,7 @@ sub _msg_page_prepare {
                        $hbuf .= "$h: $_\n" for ($eml->header_raw($h));
                }
                $ctx->{-linkify}->linkify_mids('..', \$hbuf, 1); # escapes HTML
-               $ctx->zmore($hbuf);
+               $ctx->zadd($hbuf);
                $hbuf = '';
        }
        my @irt = $eml->header_raw('In-Reply-To');
@@ -717,7 +717,7 @@ sub _msg_page_prepare {
                $hbuf .= 'References: <'.join(">\n\t<", @$refs).">\n" if @$refs;
        }
        $ctx->{-linkify}->linkify_mids('..', \$hbuf); # escapes HTML
-       $ctx->zmore($hbuf .= "\n");
+       $ctx->zadd($hbuf .= "\n");
        1;
 }
 
@@ -837,7 +837,7 @@ EOF
        $foot .= qq(<a\nhref="#R">reply</a>);
        # $skel may be big for big threads, don't append it to $foot
        $skel .= '</pre>' . ($related // '');
-       $ctx->zmore($foot, $skel .= msg_reply($ctx, $hdr));
+       $ctx->zadd($foot, $skel .= msg_reply($ctx, $hdr));
 }
 
 sub ghost_parent {

diff --git a/lib/PublicInbox/ViewDiff.pm b/lib/PublicInbox/ViewDiff.pm
index 36601910ad1ef9d5518109722c4b91ddfaa31536..95b615dc7c11a7a2084b57d9a1e1bcaac64300de 100644 (file)
--- a/lib/PublicInbox/ViewDiff.pm
+++ b/lib/PublicInbox/ViewDiff.pm
@@ -156,7 +156,7 @@ sub diff_header ($$$) {
                warn "BUG? <$$x> had no ^index line";
        }
        $$x =~ s!^diff --git!anchor1($ctx, $pb) // 'diff --git'!ems;
-       $ctx->zmore(qq(<span\nclass="head">$$x</span>));
+       $ctx->zadd(qq(<span\nclass="head">$$x</span>));
        $dctx;
 }
 
@@ -180,9 +180,9 @@ sub diff_before_or_after ($$) {
                $$x .= qq(<a href="$ch">changed</a>,);
                $$x .= ascii_html(pop @x); # $4: insertions/deletions
                # notes, commit message, etc
-               $ctx->zmore($$x .= $lnk->to_html(pop @x));
+               $ctx->zadd($$x .= $lnk->to_html(pop @x));
        } else {
-               $ctx->zmore($ctx->{-linkify}->to_html($$x));
+               $ctx->zadd($ctx->{-linkify}->to_html($$x));
        }
 }
 
@@ -220,23 +220,23 @@ sub flush_diff ($$) {
                                if (!defined($dctx)) {
                                        $after .= $s;
                                } elsif ($s =~ s/\A@@ (\S+) (\S+) @@//) {
-                                       $ctx->zmore(qq(<span\nclass="hunk">) .
+                                       $ctx->zadd(qq(<span\nclass="hunk">) .
                                                diff_hunk($dctx, $1, $2) .
                                                $lnk->to_html($s) .
                                                '</span>');
                                } elsif ($s =~ /\A\+/) { # $s may be huge
-                                       $ctx->zmore(qq(<span\nclass="add">),
+                                       $ctx->zadd(qq(<span\nclass="add">),
                                                        $lnk->to_html($s),
                                                        '</span>');
                                } elsif ($s =~ /\A-- $/sm) { # email sig starts
                                        $dctx = undef;
                                        $after .= $s;
                                } elsif ($s =~ /\A-/) { # $s may be huge
-                                       $ctx->zmore(qq(<span\nclass="del">),
+                                       $ctx->zadd(qq(<span\nclass="del">),
                                                $lnk->to_html($s),
                                                '</span>');
                                } else { # $s may be huge
-                                       $ctx->zmore($lnk->to_html($s));
+                                       $ctx->zadd($lnk->to_html($s));
                                }
                        }
                        diff_before_or_after($ctx, \$after) if !$dctx;

diff --git a/lib/PublicInbox/WwwAtomStream.pm b/lib/PublicInbox/WwwAtomStream.pm
index cdfbf3931a9491ba9e967bb230a84cdcce1ed488..1c7ae881018773ef91feaab9c37c54769f59a908 100644 (file)
--- a/lib/PublicInbox/WwwAtomStream.pm
+++ b/lib/PublicInbox/WwwAtomStream.pm
@@ -146,7 +146,7 @@ sub feed_entry {
        my $name = ascii_html(join(', ', PublicInbox::Address::names($from)));
        $email = ascii_html($email // $ctx->{ibx}->{-primary_address});
 
-       $ctx->zmore(
+       $ctx->zadd(
                (delete($ctx->{emit_header}) ? atom_header($ctx, $title) : '').
                "<entry><author><name>$name</name><email>$email</email>" .
                "</author>$title$updated" .

diff --git a/lib/PublicInbox/WwwStream.pm b/lib/PublicInbox/WwwStream.pm
index c23668a440bbcfe12c4eee3d7ffb52533c21ae1a..2a318e5e6640cc3071e5b443bb2b6df4de457dab 100644 (file)
--- a/lib/PublicInbox/WwwStream.pm
+++ b/lib/PublicInbox/WwwStream.pm
@@ -182,7 +182,7 @@ sub html_oneshot ($$;@) {
        bless $ctx, __PACKAGE__;
        $ctx->{gz} = PublicInbox::GzipFilter::gz_or_noop($res_hdr, $ctx->{env});
        $ctx->{base_url} // do {
-               $ctx->zmore(html_top($ctx));
+               $ctx->zadd(html_top($ctx));
                $ctx->{base_url} = base_url($ctx);
        };
        my $bdy = $ctx->zflush(@_[2..$#_], _html_end($ctx));
@@ -216,7 +216,7 @@ sub html_init {
        my $h = $ctx->{-res_hdr} = ['Content-Type', 'text/html; charset=UTF-8'];
        $ctx->{gz} = PublicInbox::GzipFilter::gz_or_noop($h, $ctx->{env});
        bless $ctx, __PACKAGE__;
-       $ctx->zmore(html_top($ctx));
+       $ctx->zadd(html_top($ctx));
 }
 
 1;