From: Eric Wong Date: Sun, 30 Jun 2019 17:13:30 +0000 (+0000) Subject: Merge remote-tracking branch 'origin/email-simple-mem' into master X-Git-Tag: v1.2.0~135 X-Git-Url: http://www.git.stargrave.org/?p=public-inbox.git;a=commitdiff_plain;h=ecea327e3d4386a22652fc08f71ac7d65b8f9b70;hp=0ea8f5dacfb6d6bfba4dd4be86416bdd2dc5907d Merge remote-tracking branch 'origin/email-simple-mem' into master * origin/email-simple-mem: nntp: reduce syscalls for ARTICLE and BODY mbox: split header and body processing mbox: use Email::Simple->new to do in-place modifications nntp: rework and simplify art_lookup response --- diff --git a/lib/PublicInbox/Mbox.pm b/lib/PublicInbox/Mbox.pm index 15200d3a..0c3e52fe 100644 --- a/lib/PublicInbox/Mbox.pm +++ b/lib/PublicInbox/Mbox.pm @@ -16,8 +16,8 @@ use Email::Simple; use Email::MIME::Encode; sub subject_fn ($) { - my ($simple) = @_; - my $fn = $simple->header('Subject'); + my ($hdr) = @_; + my $fn = $hdr->header('Subject'); return 'no-subject' unless defined($fn); # no need for full Email::MIME, here @@ -36,19 +36,26 @@ sub mb_stream { } # called by PSGI server as body response +# this gets called twice for every message, once to return the header, +# once to retrieve the body sub getline { my ($more) = @_; # self - my ($ctx, $id, $prev, $next, $cur) = @$more; - if ($cur) { # first - pop @$more; - return msg_str($ctx, $cur); + my ($ctx, $id, $prev, $next, $mref, $hdr) = @$more; + if ($hdr) { # first message hits this, only + pop @$more; # $hdr + return msg_hdr($ctx, $hdr); } - $cur = $next or return; + if ($mref) { # all messages hit this + pop @$more; # $mref + return msg_body($$mref); + } + my $cur = $next or return; my $ibx = $ctx->{-inbox}; $next = $ibx->over->next_by_mid($ctx->{mid}, \$id, \$prev); - @$more = ($ctx, $id, $prev, $next); # $next may be undef, here - my $mref = $ibx->msg_by_smsg($cur) or return; - msg_str($ctx, Email::Simple->new($mref)); + $mref = $ibx->msg_by_smsg($cur) or return; + $hdr = Email::Simple->new($mref)->header_obj; + @$more = ($ctx, $id, $prev, $next, $mref); # $next may be undef, here + msg_hdr($ctx, $hdr); # all but first message hits this } sub close {} # noop @@ -57,22 +64,17 @@ sub emit_raw { my ($ctx) = @_; my $mid = $ctx->{mid}; my $ibx = $ctx->{-inbox}; - my $first; - my $more; + my ($mref, $more, $id, $prev, $next); if (my $over = $ibx->over) { - my ($id, $prev); my $smsg = $over->next_by_mid($mid, \$id, \$prev) or return; - my $mref = $ibx->msg_by_smsg($smsg) or return; - $first = Email::Simple->new($mref); - my $next = $over->next_by_mid($mid, \$id, \$prev); - # $more is for ->getline - $more = [ $ctx, $id, $prev, $next, $first ] if $next; + $mref = $ibx->msg_by_smsg($smsg) or return; + $next = $over->next_by_mid($mid, \$id, \$prev); } else { - my $mref = $ibx->msg_by_mid($mid) or return; - $first = Email::Simple->new($mref); + $mref = $ibx->msg_by_mid($mid) or return; } - return unless defined $first; - my $fn = subject_fn($first); + my $hdr = Email::Simple->new($mref)->header_obj; + $more = [ $ctx, $id, $prev, $next, $mref, $hdr ]; # for ->getline + my $fn = subject_fn($hdr); my @hdr = ('Content-Type'); if ($ibx->{obfuscate}) { # obfuscation is stupid, but maybe scrapers are, too... @@ -83,12 +85,11 @@ sub emit_raw { $fn .= '.txt'; } push @hdr, 'Content-Disposition', "inline; filename=$fn"; - [ 200, \@hdr, $more ? mb_stream($more) : [ msg_str($ctx, $first) ] ]; + [ 200, \@hdr, mb_stream($more) ]; } -sub msg_str { - my ($ctx, $simple, $mid) = @_; # Email::Simple object - my $header_obj = $simple->header_obj; +sub msg_hdr ($$;$) { + my ($ctx, $header_obj, $mid) = @_; # drop potentially confusing headers, ssoma already should've dropped # Lines and Content-Length @@ -104,7 +105,7 @@ sub msg_str { 'List-Archive', "<$base>", 'List-Post', "{-primary_address}>", ); - my $crlf = $simple->crlf; + my $crlf = $header_obj->crlf; my $buf = "From mboxrd\@z Thu Jan 1 00:00:00 1970\n" . $header_obj->as_string; for (my $i = 0; $i < @append; $i += 2) { @@ -120,13 +121,13 @@ sub msg_str { $buf .= "$k: $v$crlf" if defined $v; } $buf .= $crlf; +} +sub msg_body ($) { # mboxrd quoting style # ref: http://www.qmail.org/man/man5/mbox.html - my $body = $simple->body; - $body =~ s/^(>*From )/>$1/gm; - $buf .= $body; - $buf .= "\n"; + $_[0] =~ s/^(>*From )/>$1/gm; + $_[0] .= "\n"; } sub thread_mbox { @@ -267,11 +268,13 @@ sub response { sub getline { my ($self) = @_; my $ctx = $self->{ctx} or return; + my $gz = $self->{gz}; while (my $smsg = $self->{cb}->()) { - my $msg = $ctx->{-inbox}->msg_by_smsg($smsg) or next; - $msg = Email::Simple->new($msg); - $self->{gz}->write(PublicInbox::Mbox::msg_str($ctx, $msg, - $smsg->{mid})); + my $mref = $ctx->{-inbox}->msg_by_smsg($smsg) or next; + my $h = Email::Simple->new($mref)->header_obj; + $gz->write(PublicInbox::Mbox::msg_hdr($ctx, $h, $smsg->{mid})); + $gz->write(PublicInbox::Mbox::msg_body($$mref)); + my $bref = $self->{buf}; if (length($$bref) >= 8192) { my $ret = $$bref; # copy :< diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm index 82762b1a..26bc679f 100644 --- a/lib/PublicInbox/NNTP.pm +++ b/lib/PublicInbox/NNTP.pm @@ -487,24 +487,23 @@ find_mid: found: my $smsg = $ng->over->get_art($n) or return $err; my $msg = $ng->msg_by_smsg($smsg) or return $err; - my $s = Email::Simple->new($msg); - if ($set_headers) { - set_nntp_headers($self, $s->header_obj, $ng, $n, $mid); - # must be last - $s->body_set('') if ($set_headers == 2); - } - [ $n, $mid, $s, $smsg->bytes, $smsg->lines, $ng ]; + # Email::Simple->new will modify $msg in-place as documented + # in its manpage, so what's left is the body and we won't need + # to call Email::Simple::body(), later + my $hdr = Email::Simple->new($msg)->header_obj; + set_nntp_headers($self, $hdr, $ng, $n, $mid) if $set_headers; + [ $n, $mid, $msg, $hdr ]; } -sub simple_body_write ($$) { - my ($self, $s) = @_; - my $body = $s->body; - $s->body_set(''); - $body =~ s/^\./../smg; - $body =~ s/(?{article} = $art if defined $art && $art =~ /\A[0-9]+\z/; } -sub _header ($) { - my $hdr = $_[0]->header_obj->as_string; +sub msg_hdr_write ($$$) { + my ($self, $hdr, $body_follows) = @_; + $hdr = $hdr->as_string; utf8::encode($hdr); - $hdr =~ s/(? article retrieved - head and body follow"); - msg_more($self, _header($s)); - msg_more($self, "\r\n"); - simple_body_write($self, $s); + msg_hdr_write($self, $hdr, 1); + msg_body_write($self, $msg); } sub cmd_head ($;$) { my ($self, $art) = @_; my $r = art_lookup($self, $art, 2); return $r unless ref $r; - my ($n, $mid, $s) = @$r; + my ($n, $mid, undef, $hdr) = @$r; set_art($self, $art); more($self, "221 $n <$mid> article retrieved - head follows"); - msg_more($self, _header($s)); + msg_hdr_write($self, $hdr, 0); '.' } @@ -554,17 +553,17 @@ sub cmd_body ($;$) { my ($self, $art) = @_; my $r = art_lookup($self, $art, 0); return $r unless ref $r; - my ($n, $mid, $s) = @$r; + my ($n, $mid, $msg) = @$r; set_art($self, $art); more($self, "222 $n <$mid> article retrieved - body follows"); - simple_body_write($self, $s); + msg_body_write($self, $msg); } sub cmd_stat ($;$) { my ($self, $art) = @_; my $r = art_lookup($self, $art, 0); return $r unless ref $r; - my ($n, $mid, undef) = @$r; + my ($n, $mid) = @$r; set_art($self, $art); "223 $n <$mid> article retrieved - request text separately"; } @@ -792,7 +791,7 @@ sub hdr_mid_prefix ($$$$$) { } sub hdr_mid_response ($$$$$$) { - my ($self, $xhdr, $ng, $n, $mid, $v) = @_; # r: art_lookup result + my ($self, $xhdr, $ng, $n, $mid, $v) = @_; my $res = ''; if ($xhdr) { $res .= r221 . "\r\n";