From: Eric Wong Date: Thu, 27 Jun 2019 09:23:39 +0000 (+0000) Subject: mbox: split header and body processing X-Git-Tag: v1.2.0~135^2~1 X-Git-Url: http://www.git.stargrave.org/?a=commitdiff_plain;h=71ea9961786fa14ea0a67200847bac5a76abf751;hp=018295ad147ecbe9759d46bc8d74a776d4ad382f;p=public-inbox.git mbox: split header and body processing When dealing with ~30MB messages, we can save another ~30MB by splitting the header and body processing and not appending the body string back to the header. We'll rely on buffering in gzip or kernel (via MSG_MORE) to prevent silly packet sizes. --- diff --git a/lib/PublicInbox/Mbox.pm b/lib/PublicInbox/Mbox.pm index 1bf71c60..0c3e52fe 100644 --- a/lib/PublicInbox/Mbox.pm +++ b/lib/PublicInbox/Mbox.pm @@ -16,8 +16,8 @@ use Email::Simple; use Email::MIME::Encode; sub subject_fn ($) { - my ($simple) = @_; - my $fn = $simple->header('Subject'); + my ($hdr) = @_; + my $fn = $hdr->header('Subject'); return 'no-subject' unless defined($fn); # no need for full Email::MIME, here @@ -36,20 +36,26 @@ sub mb_stream { } # called by PSGI server as body response +# this gets called twice for every message, once to return the header, +# once to retrieve the body sub getline { my ($more) = @_; # self - my ($ctx, $id, $prev, $next, $cur, $mref) = @$more; - if ($mref) { # first - pop @$more; - pop @$more; - return msg_str($ctx, $cur, $mref); + my ($ctx, $id, $prev, $next, $mref, $hdr) = @$more; + if ($hdr) { # first message hits this, only + pop @$more; # $hdr + return msg_hdr($ctx, $hdr); } - $cur = $next or return; + if ($mref) { # all messages hit this + pop @$more; # $mref + return msg_body($$mref); + } + my $cur = $next or return; my $ibx = $ctx->{-inbox}; $next = $ibx->over->next_by_mid($ctx->{mid}, \$id, \$prev); - @$more = ($ctx, $id, $prev, $next); # $next may be undef, here $mref = $ibx->msg_by_smsg($cur) or return; - msg_str($ctx, Email::Simple->new($mref), $mref); + $hdr = Email::Simple->new($mref)->header_obj; + @$more = ($ctx, $id, $prev, $next, $mref); # $next may be undef, here + msg_hdr($ctx, $hdr); # all but first message hits this } sub close {} # noop @@ -58,21 +64,17 @@ sub emit_raw { my ($ctx) = @_; my $mid = $ctx->{mid}; my $ibx = $ctx->{-inbox}; - my ($first, $mref, $more); + my ($mref, $more, $id, $prev, $next); if (my $over = $ibx->over) { - my ($id, $prev); my $smsg = $over->next_by_mid($mid, \$id, \$prev) or return; $mref = $ibx->msg_by_smsg($smsg) or return; - $first = Email::Simple->new($mref); - my $next = $over->next_by_mid($mid, \$id, \$prev); - # $more is for ->getline - $more = [ $ctx, $id, $prev, $next, $first, $mref ] if $next; + $next = $over->next_by_mid($mid, \$id, \$prev); } else { $mref = $ibx->msg_by_mid($mid) or return; - $first = Email::Simple->new($mref); } - return unless defined $first; - my $fn = subject_fn($first); + my $hdr = Email::Simple->new($mref)->header_obj; + $more = [ $ctx, $id, $prev, $next, $mref, $hdr ]; # for ->getline + my $fn = subject_fn($hdr); my @hdr = ('Content-Type'); if ($ibx->{obfuscate}) { # obfuscation is stupid, but maybe scrapers are, too... @@ -83,13 +85,11 @@ sub emit_raw { $fn .= '.txt'; } push @hdr, 'Content-Disposition', "inline; filename=$fn"; - [ 200, \@hdr, - $more ? mb_stream($more) : [ msg_str($ctx, $first, $mref) ] ]; + [ 200, \@hdr, mb_stream($more) ]; } -sub msg_str { - my ($ctx, $simple, $mref, $mid) = @_; # simple - Email::Simple object - my $header_obj = $simple->header_obj; +sub msg_hdr ($$;$) { + my ($ctx, $header_obj, $mid) = @_; # drop potentially confusing headers, ssoma already should've dropped # Lines and Content-Length @@ -121,12 +121,13 @@ sub msg_str { $buf .= "$k: $v$crlf" if defined $v; } $buf .= $crlf; +} +sub msg_body ($) { # mboxrd quoting style # ref: http://www.qmail.org/man/man5/mbox.html - $$mref =~ s/^(>*From )/>$1/gm; - $buf .= $$mref; - $buf .= "\n"; + $_[0] =~ s/^(>*From )/>$1/gm; + $_[0] .= "\n"; } sub thread_mbox { @@ -267,11 +268,13 @@ sub response { sub getline { my ($self) = @_; my $ctx = $self->{ctx} or return; + my $gz = $self->{gz}; while (my $smsg = $self->{cb}->()) { my $mref = $ctx->{-inbox}->msg_by_smsg($smsg) or next; - my $s = Email::Simple->new($mref); - $self->{gz}->write(PublicInbox::Mbox::msg_str($ctx, $s, $mref, - $smsg->{mid})); + my $h = Email::Simple->new($mref)->header_obj; + $gz->write(PublicInbox::Mbox::msg_hdr($ctx, $h, $smsg->{mid})); + $gz->write(PublicInbox::Mbox::msg_body($$mref)); + my $bref = $self->{buf}; if (length($$bref) >= 8192) { my $ret = $$bref; # copy :<