From: Eric Wong Date: Thu, 10 Jan 2019 21:41:55 +0000 (+0000) Subject: Merge commit 'mem' X-Git-Tag: v1.2.0~411 X-Git-Url: http://www.git.stargrave.org/?p=public-inbox.git;a=commitdiff_plain;h=b0e5062d43a96372801713ef78a78d6a1bc852bc;hp=f6fea847bf71e5cc247497c95cdb73ccc74466d2 Merge commit 'mem' * commit 'mem': view: more culling for search threads over: cull unneeded fields for get_thread searchmsg: remove unused fields for PSGI in Xapian results searchview: drop unused {seen} hashref searchmsg: remove Xapian::Document field searchmsg: get rid of termlist scanning for mid httpd: remove psgix.harakiri reference --- diff --git a/lib/PublicInbox/HTTPD.pm b/lib/PublicInbox/HTTPD.pm index 8cf365d0..38517710 100644 --- a/lib/PublicInbox/HTTPD.pm +++ b/lib/PublicInbox/HTTPD.pm @@ -29,7 +29,6 @@ sub new { 'psgi.run_once' => Plack::Util::FALSE, 'psgi.multithread' => Plack::Util::FALSE, 'psgi.multiprocess' => Plack::Util::TRUE, - 'psgix.harakiri'=> Plack::Util::FALSE, 'psgix.input.buffered' => Plack::Util::TRUE, # XXX unstable API! diff --git a/lib/PublicInbox/Inbox.pm b/lib/PublicInbox/Inbox.pm index 73f5761a..d57e46d2 100644 --- a/lib/PublicInbox/Inbox.pm +++ b/lib/PublicInbox/Inbox.pm @@ -302,8 +302,9 @@ sub smsg_by_mid ($$) { my ($self, $mid) = @_; my $srch = search($self) or return; # favor the Message-ID we used for the NNTP article number: - my $num = mid2num($self, $mid); - defined $num ? $srch->lookup_article($num) : undef; + defined(my $num = mid2num($self, $mid)) or return; + my $smsg = $srch->lookup_article($num) or return; + PublicInbox::SearchMsg::psgi_cull($smsg); } sub msg_by_mid ($$;$) { diff --git a/lib/PublicInbox/Over.pm b/lib/PublicInbox/Over.pm index dda1e6d0..598c9fcb 100644 --- a/lib/PublicInbox/Over.pm +++ b/lib/PublicInbox/Over.pm @@ -40,13 +40,16 @@ sub disconnect { $_[0]->{dbh} = undef } sub connect { $_[0]->{dbh} ||= $_[0]->dbh_new } -sub load_from_row { - my ($smsg) = @_; +sub load_from_row ($;$) { + my ($smsg, $cull) = @_; bless $smsg, 'PublicInbox::SearchMsg'; if (defined(my $data = delete $smsg->{ddd})) { $data = uncompress($data); utf8::decode($data); - $smsg->load_from_data($data); + PublicInbox::SearchMsg::load_from_data($smsg, $data); + + # saves over 600K for 1000+ message threads + PublicInbox::SearchMsg::psgi_cull($smsg) if $cull; } $smsg } @@ -57,7 +60,8 @@ sub do_get { my $lim = (($opts->{limit} || 0) + 0) || DEFAULT_LIMIT; $sql .= "LIMIT $lim"; my $msgs = $dbh->selectall_arrayref($sql, { Slice => {} }, @args); - load_from_row($_) for @$msgs; + my $cull = $opts->{cull}; + load_from_row($_, $cull) for @$msgs; $msgs } @@ -82,6 +86,7 @@ sub nothing () { wantarray ? (0, []) : [] }; sub get_thread { my ($self, $mid, $prev) = @_; my $dbh = $self->connect; + my $opts = { cull => 1 }; my $id = $dbh->selectrow_array(<<'', undef, $mid); SELECT id FROM msgid WHERE mid = ? LIMIT 1 @@ -109,7 +114,7 @@ SELECT tid,sid FROM over WHERE num = ? LIMIT 1 my $cols = 'num,ts,ds,ddd'; unless (wantarray) { - return do_get($self, <<"", {}, $tid, $sid, $num); + return do_get($self, <<"", $opts, $tid, $sid, $num); SELECT $cols FROM over WHERE $cond_all ORDER BY $sort_col ASC @@ -123,14 +128,14 @@ SELECT COUNT(num) FROM over WHERE $cond_all # giant thread, prioritize strict (tid) matches and throw # in the loose (sid) matches at the end - my $msgs = do_get($self, <<"", {}, $tid, $num); + my $msgs = do_get($self, <<"", $opts, $tid, $num); SELECT $cols FROM over WHERE tid = ? AND num > ? ORDER BY $sort_col ASC # do we have room for loose matches? get the most recent ones, first: my $lim = DEFAULT_LIMIT - scalar(@$msgs); if ($lim > 0) { - my $opts = { limit => $lim }; + $opts->{limit} = $lim; my $loose = do_get($self, <<"", $opts, $tid, $sid, $num); SELECT $cols FROM over WHERE tid != ? AND sid = ? AND num > ? ORDER BY $sort_col DESC diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm index cc1ac56b..8810fe76 100644 --- a/lib/PublicInbox/SearchIdx.pm +++ b/lib/PublicInbox/SearchIdx.pm @@ -282,7 +282,7 @@ sub index_body ($$$) { sub add_xapian ($$$$$) { my ($self, $mime, $num, $oid, $mids, $mid0) = @_; my $smsg = PublicInbox::SearchMsg->new($mime); - my $doc = $smsg->{doc}; + my $doc = Search::Xapian::Document->new; my $subj = $smsg->subject; add_val($doc, PublicInbox::Search::TS(), $smsg->ts); my @ds = gmtime($smsg->ds); @@ -439,8 +439,8 @@ sub remove_by_oid { for (; $head != $tail; $head->inc) { my $docid = $head->get_docid; my $doc = $db->get_document($docid); - my $smsg = PublicInbox::SearchMsg->wrap($doc, $mid); - $smsg->load_expand; + my $smsg = PublicInbox::SearchMsg->wrap($mid); + $smsg->load_expand($doc); if ($smsg->{blob} eq $oid) { push(@delete, $docid); } diff --git a/lib/PublicInbox/SearchMsg.pm b/lib/PublicInbox/SearchMsg.pm index 32cceb25..ceb6edad 100644 --- a/lib/PublicInbox/SearchMsg.pm +++ b/lib/PublicInbox/SearchMsg.pm @@ -15,20 +15,18 @@ use Time::Local qw(timegm); sub new { my ($class, $mime) = @_; - my $doc = Search::Xapian::Document->new; - bless { doc => $doc, mime => $mime }, $class; + bless { mime => $mime }, $class; } sub wrap { - my ($class, $doc, $mid) = @_; - bless { doc => $doc, mime => undef, mid => $mid }, $class; + my ($class, $mid) = @_; + bless { mid => $mid }, $class; } sub get { my ($class, $head, $db, $mid) = @_; my $doc_id = $head->get_docid; - my $doc = $db->get_document($doc_id); - load_expand(wrap($class, $doc, $mid)) + load_expand(wrap($class, $mid), $db->get_document($doc_id)); } sub get_val ($$) { @@ -61,19 +59,21 @@ sub load_from_data ($$) { # To: and Cc: are stored to optimize HDR/XHDR in NNTP since # some NNTP clients will use that for message displays. + # NNTP only, and only stored in Over(view), not Xapian $self->{to}, $self->{cc}, $self->{blob}, $self->{mid}, + + # NNTP only $self->{bytes}, $self->{lines} ) = split(/\n/, $_[1]); } sub load_expand { - my ($self) = @_; - my $doc = $self->{doc}; + my ($self, $doc) = @_; my $data = $doc->get_data or return; $self->{ts} = get_val($doc, PublicInbox::Search::TS()); my $dt = get_val($doc, PublicInbox::Search::DT()); @@ -84,10 +84,21 @@ sub load_expand { $self; } +sub psgi_cull ($) { + my ($self) = @_; + from_name($self); # fill in {from_name} so we can delete {from} + + # drop NNTP-only fields which aren't relevant to PSGI results: + # saves ~80K on a 200 item search result: + delete @$self{qw(from ts to cc bytes lines)}; + $self; +} + +# Only called by PSGI interface, not NNTP sub load_doc { my ($class, $doc) = @_; - my $self = bless { doc => $doc }, $class; - $self->load_expand; + my $self = bless {}, $class; + psgi_cull(load_expand($self, $doc)); } # :bytes and :lines metadata in RFC 3977 @@ -159,29 +170,15 @@ sub references { defined $x ? $x : ''; } -sub _get_term_val ($$$) { - my ($self, $pfx, $re) = @_; - my $doc = $self->{doc}; - my $end = $doc->termlist_end; - my $i = $doc->termlist_begin; - $i->skip_to($pfx); - if ($i != $end) { - my $val = $i->get_termname; - $val =~ s/$re// and return $val; - } - undef; -} - sub mid ($;$) { my ($self, $mid) = @_; if (defined $mid) { $self->{mid} = $mid; - } elsif (my $rv = $self->{mid}) { + } elsif (defined(my $rv = $self->{mid})) { $rv; - } elsif ($self->{doc}) { - $self->{mid} = _get_term_val($self, 'Q', qr/\AQ/); } else { + die "NO {mime} for mid\n" unless $self->{mime}; $self->_extract_mid; # v1 w/o Xapian } } diff --git a/lib/PublicInbox/SearchThread.pm b/lib/PublicInbox/SearchThread.pm index be290980..931bd579 100644 --- a/lib/PublicInbox/SearchThread.pm +++ b/lib/PublicInbox/SearchThread.pm @@ -53,6 +53,11 @@ sub _add_message ($$) { my $this = _get_cont_for_id($id_table, $smsg->{mid}); $this->{smsg} = $smsg; + # saves around 4K across 1K messages + # TODO: move this to a more appropriate place, breaks tests + # if we do it during psgi_cull + delete $smsg->{num}; + # B. For each element in the message's References field: defined(my $refs = $smsg->{references}) or return; diff --git a/lib/PublicInbox/SearchView.pm b/lib/PublicInbox/SearchView.pm index e47dcfd0..c33caec9 100644 --- a/lib/PublicInbox/SearchView.pm +++ b/lib/PublicInbox/SearchView.pm @@ -240,7 +240,6 @@ sub mset_thread { $ctx->{pct} = \%pct; $ctx->{prev_attr} = ''; $ctx->{prev_level} = 0; - $ctx->{seen} = {}; $ctx->{s_nr} = scalar(@$msgs).'+ results'; # reduce hash lookups in skel_dump diff --git a/lib/PublicInbox/View.pm b/lib/PublicInbox/View.pm index 5ddb8425..cd125e00 100644 --- a/lib/PublicInbox/View.pm +++ b/lib/PublicInbox/View.pm @@ -219,7 +219,10 @@ sub index_entry { $rv .= _th_index_lite($mid_raw, \$irt, $id, $ctx); my @tocc; my $ds = $smsg->ds; # for v1 non-Xapian/SQLite users - my $mime = delete $smsg->{mime}; # critical to memory use + # deleting {mime} is critical to memory use, + # the rest of the fields saves about 400K as we iterate across 1K msgs + my ($mime) = delete @$smsg{qw(mime ds ts blob subject)}; + my $hdr = $mime->header_obj; my $from = _hdr_names_html($hdr, 'From'); obfuscate_addrs($obfs_ibx, $from) if $obfs_ibx; @@ -311,7 +314,10 @@ sub _th_index_lite { my $nr_s = 0; my $siblings; if (my $smsg = $node->{smsg}) { - ($$irt) = (($smsg->{references} || '') =~ m/<([^>]+)>\z/); + # delete saves about 200KB on a 1K message thread + if (my $refs = delete $smsg->{references}) { + ($$irt) = ($refs =~ m/<([^>]+)>\z/); + } } my $irt_map = $mapping->{$$irt} if defined $$irt; if (defined $irt_map) { diff --git a/t/search.t b/t/search.t index 3c758e68..6415a644 100644 --- a/t/search.t +++ b/t/search.t @@ -341,7 +341,7 @@ $ibx->with_umask(sub { is(scalar(@$res), 1, "searched $pfx successfully for From:"); foreach my $smsg (@$res) { - like($smsg->from, qr/Laggy Sender/, + like($smsg->from_name, qr/Laggy Sender/, "From appears with $pfx"); } } @@ -358,16 +358,18 @@ $ibx->with_umask(sub { $res = $ro->query('q:theatre'); is(scalar(@$res), 1, 'only one quoted body'); - like($res->[0]->from, qr/\AQuoter/, 'got quoted body') if scalar(@$res); + like($res->[0]->from_name, qr/\AQuoter/, + 'got quoted body') if (scalar(@$res)); $res = $ro->query('nq:theatre'); is(scalar @$res, 1, 'only one non-quoted body'); - like($res->[0]->from, qr/\ANon-Quoter/, 'got non-quoted body') if scalar(@$res); + like($res->[0]->from_name, qr/\ANon-Quoter/, + 'got non-quoted body') if (scalar(@$res)); foreach my $pfx (qw(b: bs:)) { $res = $ro->query($pfx . 'theatre'); is(scalar @$res, 2, "searched both bodies for $pfx"); - like($res->[0]->from, qr/\ANon-Quoter/, + like($res->[0]->from_name, qr/\ANon-Quoter/, "non-quoter first for $pfx") if scalar(@$res); } }