view: reduce memory usage when displaying large threads

We want to minimize the time any large objects or strings
are referenced.  We can do threading entirely from the
mini_mime-generated messages and lazilly load full messages
when rendering the display.

diff --git a/lib/PublicInbox/View.pm b/lib/PublicInbox/View.pm
index 2ba51182dfcc39f3fe2126885406cc28f8d271f4..391e3ad79b0dad9c01ce5df4f738b2e5ede34719 100644 (file)
--- a/lib/PublicInbox/View.pm
+++ b/lib/PublicInbox/View.pm
@@ -146,7 +146,7 @@ sub thread_html {
        my $mid = mid_compressed($ctx->{mid});
        my $res = $srch->get_thread($mid);
        my $rv = '';
-       my $msgs = load_results($ctx, $res);
+       my $msgs = load_results($res);
        my $nr = scalar @$msgs;
        return $rv if $nr == 0;
        require PublicInbox::Thread;
@@ -154,7 +154,11 @@ sub thread_html {
        $th->thread;
        $th->order(*PublicInbox::Thread::sort_ts);
        my $state = [ $srch, { root_anchor => anchor_for($mid) }, undef, 0 ];
-       thread_entry(\$rv, $state, $_, 0) for $th->rootset;
+       {
+               require PublicInbox::GitCatFile;
+               my $git = PublicInbox::GitCatFile->new($ctx->{git_dir});
+               thread_entry(\$rv, $git, $state, $_, 0) for $th->rootset;
+       }
        my $final_anchor = $state->[3];
        my $next = "<a\nid=\"s$final_anchor\">";
 
@@ -173,7 +177,7 @@ sub subject_path_html {
        my $path = $ctx->{subject_path};
        my $res = $srch->get_subject_path($path);
        my $rv = '';
-       my $msgs = load_results($ctx, $res);
+       my $msgs = load_results($res);
        my $nr = scalar @$msgs;
        return $rv if $nr == 0;
        require PublicInbox::Thread;
@@ -181,7 +185,11 @@ sub subject_path_html {
        $th->thread;
        $th->order(*PublicInbox::Thread::sort_ts);
        my $state = [ $srch, { root_anchor => 'dummy' }, undef, 0 ];
-       thread_entry(\$rv, $state, $_, 0) for $th->rootset;
+       {
+               require PublicInbox::GitCatFile;
+               my $git = PublicInbox::GitCatFile->new($ctx->{git_dir});
+               thread_entry(\$rv, $git, $state, $_, 0) for $th->rootset;
+       }
        my $final_anchor = $state->[3];
        my $next = "<a\nid=\"s$final_anchor\">end of thread</a>\n";
 
@@ -197,7 +205,10 @@ sub index_walk {
        my $ct = $part->content_type;
 
        # account for filter bugs...
-       return '' if defined $ct && $ct =~ m!\btext/[xh]+tml\b!i;
+       if (defined $ct && $ct =~ m!\btext/[xh]+tml\b!i) {
+               $part->body_set('');
+               return '';
+       }
 
        my $enc = enc_for($ct, $enc_msg);
 
@@ -224,7 +235,9 @@ sub index_walk {
                # kill per-line trailing whitespace
                $s =~ s/[ \t]+$//sgm;
 
-               $rv .= $s . "\n";
+               $rv .= $s;
+               $s = undef;
+               $rv .= "\n";
        }
        $rv;
 }
@@ -335,10 +348,13 @@ sub flush_quote {
 sub add_text_body {
        my ($enc, $part, $part_nr, $full_pfx) = @_;
        my $n = 0;
-       my $s = ascii_html($enc->decode($part->body));
+       my $nr = 0;
+       my $s = $part->body;
+       $part->body_set('');
+       $s = $enc->decode($s);
+       $s = ascii_html($s);
        my @lines = split(/\n/, $s);
        $s = '';
-       my $nr = 0;
        my @quot;
        while (defined(my $cur = shift @lines)) {
                if ($cur !~ /^&gt;/) {
@@ -538,10 +554,10 @@ sub simple_dump {
 
 sub thread_followups {
        my ($dst, $root, $res) = @_;
-       my @msgs = map { $_->mini_mime } @{$res->{msgs}};
+       my $msgs = load_results($res);
        require PublicInbox::Thread;
        $root->header_set('X-PI-TS', '0');
-       my $th = PublicInbox::Thread->new($root, @msgs);
+       my $th = PublicInbox::Thread->new($root, @$msgs);
        $th->thread;
        $th->order(*PublicInbox::Thread::sort_ts);
        my $srch = $res->{srch};
@@ -559,43 +575,35 @@ sub thread_html_head {
 }
 
 sub thread_entry {
-       my ($dst, $state, $node, $level) = @_;
+       my ($dst, $git, $state, $node, $level) = @_;
        # $state = [ $search_res, $seen, undef, 0 (msg_nr) ];
        # $seen is overloaded with 3 types of fields:
        #       1) "root_anchor" => anchor_for(Message-ID),
        #       2) seen subject hashes: sha1(subject) => 1
        #       3) anchors hashes: "#$sha1_hex" (same as $seen in index_entry)
        if (my $mime = $node->message) {
-               if (length($$dst) == 0) {
-                       $$dst .= thread_html_head($mime);
+
+               # lazy load the full message from mini_mime:
+               my $path = mid2path(mid_clean($mime->header('Message-ID')));
+               $mime = eval { Email::MIME->new($git->cat_file("HEAD:$path")) };
+               if ($mime) {
+                       if (length($$dst) == 0) {
+                               $$dst .= thread_html_head($mime);
+                       }
+                       $$dst .= index_entry(undef, $mime, $level, $state);
                }
-               $$dst .= index_entry(undef, $mime, $level, $state);
        }
-       thread_entry($dst, $state, $node->child, $level + 1) if $node->child;
-       thread_entry($dst, $state, $node->next, $level) if $node->next;
+       my $cur;
+       $cur = $node->child and
+               thread_entry($dst, $git, $state, $cur, $level + 1);
+       $cur = $node->next and
+               thread_entry($dst, $git, $state, $cur, $level);
 }
 
 sub load_results {
-       my ($ctx, $res) = @_;
-
-       require PublicInbox::GitCatFile;
-       my $git = PublicInbox::GitCatFile->new($ctx->{git_dir});
-       my @msgs;
-       while (my $smsg = shift @{$res->{msgs}}) {
-               my $m = $smsg->mid;
-               my $path = mid2path($m);
-
-               # FIXME: duplicated code from Feed.pm
-               my $mime = eval {
-                       my $str = $git->cat_file("HEAD:$path");
-                       Email::MIME->new($str);
-               };
-               unless ($@) {
-                       $mime->header_set('X-PI-TS', msg_timestamp($mime));
-                       push @msgs, $mime;
-               }
-       }
-       \@msgs;
+       my ($res) = @_;
+
+       [ map { $_->mini_mime } @{delete $res->{msgs}} ];
 }
 
 sub msg_timestamp {