Merge commit 'mem'

* commit 'mem':
  view: more culling for search threads
  over: cull unneeded fields for get_thread
  searchmsg: remove unused fields for PSGI in Xapian results
  searchview: drop unused {seen} hashref
  searchmsg: remove Xapian::Document field
  searchmsg: get rid of termlist scanning for mid
  httpd: remove psgix.harakiri reference

diff --combined lib/PublicInbox/HTTPD.pm
index 8cf365d04ced13658fab55b0fa69929bdc2c9d70,905afbb89b345404135733c2edbd0dc1cf44a31a..38517710a414d00ee36d21b069de90953ffa4b08
--- 1/lib/PublicInbox/HTTPD.pm
--- 2/lib/PublicInbox/HTTPD.pm
+++ b/lib/PublicInbox/HTTPD.pm
@@@ -1,8 -1,6 +1,8 @@@
  # Copyright (C) 2016-2018 all contributors <meta@public-inbox.org>
  # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
  
 +# wraps a listen socket for HTTP and links it to the PSGI app in
 +# public-inbox-httpd
  package PublicInbox::HTTPD;
  use strict;
  use warnings;
@@@ -29,7 -27,6 +29,6 @@@ sub new 
                'psgi.run_once'  => Plack::Util::FALSE,
                'psgi.multithread' => Plack::Util::FALSE,
                'psgi.multiprocess' => Plack::Util::TRUE,
-               'psgix.harakiri'=> Plack::Util::FALSE,
                'psgix.input.buffered' => Plack::Util::TRUE,
  
                # XXX unstable API!

diff --combined lib/PublicInbox/SearchIdx.pm
index cc1ac56b933d65998bdc915711658ea5c6b4f06e,7a8ebf356efe7cb2910cd5ee5f7e14d10a423f10..8810fe76450dde64728d715e4e63f14b3c6bb741
--- 1/lib/PublicInbox/SearchIdx.pm
--- 2/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@@ -282,7 -282,7 +282,7 @@@ sub index_body ($$$) 
  sub add_xapian ($$$$$) {
        my ($self, $mime, $num, $oid, $mids, $mid0) = @_;
        my $smsg = PublicInbox::SearchMsg->new($mime);
-       my $doc = $smsg->{doc};
+       my $doc = Search::Xapian::Document->new;
        my $subj = $smsg->subject;
        add_val($doc, PublicInbox::Search::TS(), $smsg->ts);
        my @ds = gmtime($smsg->ds);
@@@ -439,8 -439,8 +439,8 @@@ sub remove_by_oid 
        for (; $head != $tail; $head->inc) {
                my $docid = $head->get_docid;
                my $doc = $db->get_document($docid);
-               my $smsg = PublicInbox::SearchMsg->wrap($doc, $mid);
-               $smsg->load_expand;
+               my $smsg = PublicInbox::SearchMsg->wrap($mid);
+               $smsg->load_expand($doc);
                if ($smsg->{blob} eq $oid) {
                        push(@delete, $docid);
                }
@@@ -646,7 -646,6 +646,7 @@@ sub _git_log 
                                --raw -r --no-abbrev/, $range);
  }
  
 +# --is-ancestor requires git 1.8.0+
  sub is_ancestor ($$$) {
        my ($git, $cur, $tip) = @_;
        return 0 unless $git->check($cur);

diff --combined lib/PublicInbox/SearchMsg.pm
index 32cceb25065cf798ba497f8ea06f5f60c8f02c5f,722a1b9428fba760591452f67a044a1af8ba812d..ceb6edadded21431bf2ca3793b254ee0b1fcd7b7
--- 1/lib/PublicInbox/SearchMsg.pm
--- 2/lib/PublicInbox/SearchMsg.pm
+++ b/lib/PublicInbox/SearchMsg.pm
@@@ -3,8 -3,6 +3,8 @@@
  # based on notmuch, but with no concept of folders, files or flags
  #
  # Wraps a document inside our Xapian search index.
 +# There may be many of these objects loaded in memory at once
 +# for large threads in our WWW UI.
  package PublicInbox::SearchMsg;
  use strict;
  use warnings;
@@@ -15,20 -13,18 +15,18 @@@ use Time::Local qw(timegm)
  
  sub new {
        my ($class, $mime) = @_;
-       my $doc = Search::Xapian::Document->new;
-       bless { doc => $doc, mime => $mime }, $class;
+       bless { mime => $mime }, $class;
  }
  
  sub wrap {
-       my ($class, $doc, $mid) = @_;
-       bless { doc => $doc, mime => undef, mid => $mid }, $class;
+       my ($class, $mid) = @_;
+       bless { mid => $mid }, $class;
  }
  
  sub get {
        my ($class, $head, $db, $mid) = @_;
        my $doc_id = $head->get_docid;
-       my $doc = $db->get_document($doc_id);
-       load_expand(wrap($class, $doc, $mid))
+       load_expand(wrap($class, $mid), $db->get_document($doc_id));
  }
  
  sub get_val ($$) {
@@@ -61,19 -57,21 +59,21 @@@ sub load_from_data ($$) 
  
                # To: and Cc: are stored to optimize HDR/XHDR in NNTP since
                # some NNTP clients will use that for message displays.
+               # NNTP only, and only stored in Over(view), not Xapian
                $self->{to},
                $self->{cc},
  
                $self->{blob},
                $self->{mid},
+ 
+               # NNTP only
                $self->{bytes},
                $self->{lines}
        ) = split(/\n/, $_[1]);
  }
  
  sub load_expand {
-       my ($self) = @_;
-       my $doc = $self->{doc};
+       my ($self, $doc) = @_;
        my $data = $doc->get_data or return;
        $self->{ts} = get_val($doc, PublicInbox::Search::TS());
        my $dt = get_val($doc, PublicInbox::Search::DT());
@@@ -84,10 -82,21 +84,21 @@@
        $self;
  }
  
+ sub psgi_cull ($) {
+       my ($self) = @_;
+       from_name($self); # fill in {from_name} so we can delete {from}
+ 
+       # drop NNTP-only fields which aren't relevant to PSGI results:
+       # saves ~80K on a 200 item search result:
+       delete @$self{qw(from ts to cc bytes lines)};
+       $self;
+ }
+ 
+ # Only called by PSGI interface, not NNTP
  sub load_doc {
        my ($class, $doc) = @_;
-       my $self = bless { doc => $doc }, $class;
-       $self->load_expand;
+       my $self = bless {}, $class;
+       psgi_cull(load_expand($self, $doc));
  }
  
  # :bytes and :lines metadata in RFC 3977
@@@ -159,29 -168,15 +170,15 @@@ sub references 
        defined $x ? $x : '';
  }
  
- sub _get_term_val ($$$) {
-       my ($self, $pfx, $re) = @_;
-       my $doc = $self->{doc};
-       my $end = $doc->termlist_end;
-       my $i = $doc->termlist_begin;
-       $i->skip_to($pfx);
-       if ($i != $end) {
-               my $val = $i->get_termname;
-               $val =~ s/$re// and return $val;
-       }
-       undef;
- }
- 
  sub mid ($;$) {
        my ($self, $mid) = @_;
  
        if (defined $mid) {
                $self->{mid} = $mid;
-       } elsif (my $rv = $self->{mid}) {
+       } elsif (defined(my $rv = $self->{mid})) {
                $rv;
-       } elsif ($self->{doc}) {
-               $self->{mid} = _get_term_val($self, 'Q', qr/\AQ/);
        } else {
+               die "NO {mime} for mid\n" unless $self->{mime};
                $self->_extract_mid; # v1 w/o Xapian
        }
  }