pop3: reduce memory use while generating the mailbox cache

[public-inbox.git] / lib / PublicInbox / POP3.pm

diff --git a/lib/PublicInbox/POP3.pm b/lib/PublicInbox/POP3.pm
index 741b5e58a90949854fdcf4331d2e3ad8e89bcfdf..203c91a6b3cff4f1697ffc8677b997d7a7fd3caf 100644 (file)
--- a/lib/PublicInbox/POP3.pm
+++ b/lib/PublicInbox/POP3.pm
@@ -55,45 +55,6 @@ sub out ($$;@) {
        printf { $self->{pop3d}->{out} } $fmt."\n", @args;
 }
 
-sub long_step {
-       my ($self) = @_;
-       # wbuf is unset or empty, here; {long} may add to it
-       my ($fd, $cb, $t0, @args) = @{$self->{long_cb}};
-       my $more = eval { $cb->($self, @args) };
-       if ($@ || !$self->{sock}) { # something bad happened...
-               delete $self->{long_cb};
-               my $elapsed = now() - $t0;
-               if ($@) {
-                       err($self,
-                           "%s during long response[$fd] - %0.6f",
-                           $@, $elapsed);
-               }
-               out($self, " deferred[$fd] aborted - %0.6f", $elapsed);
-               $self->close;
-       } elsif ($more) { # $self->{wbuf}:
-               # control passed to ibx_async_cat if $more == \undef
-               requeue_once($self) if !ref($more);
-       } else { # all done!
-               delete $self->{long_cb};
-               my $elapsed = now() - $t0;
-               my $fd = fileno($self->{sock});
-               out($self, " deferred[$fd] done - %0.6f", $elapsed);
-               my $wbuf = $self->{wbuf}; # do NOT autovivify
-               $self->requeue unless $wbuf && @$wbuf;
-       }
-}
-
-sub long_response ($$;@) {
-       my ($self, $cb, @args) = @_; # cb returns true if more, false if done
-       my $sock = $self->{sock} or return;
-       # make sure we disable reading during a long response,
-       # clients should not be sending us stuff and making us do more
-       # work while we are stream a response to them
-       $self->{long_cb} = [ fileno($sock), $cb, now(), @args ];
-       long_step($self); # kick off!
-       undef;
-}
-
 sub do_greet {
        my ($self) = @_;
        my $s = $self->{salt} = sprintf('%x.%x', int(rand(0x7fffffff)), time);
@@ -187,12 +148,21 @@ sub _stat_cache ($) {
        my ($self) = @_;
        my ($beg, $end) = (($self->{uid_dele} // -1) + 1, $self->{uid_max});
        PublicInbox::IMAP::uid_clamp($self, \$beg, \$end);
-       my $opt = { limit => PublicInbox::IMAP::UID_SLICE };
-       my $m = $self->{ibx}->over(1)->do_get(<<'', $opt, $beg, $end);
+       my (@cache, $m);
+       my $sth = $self->{ibx}->over(1)->dbh->prepare_cached(<<'', undef, 1);
 SELECT num,ddd FROM over WHERE num >= ? AND num <= ?
 ORDER BY num ASC
 
-       [ map { ($_->{num}, $_->{bytes} + 0, $_->{blob}) } @$m ];
+       $sth->execute($beg, $end);
+       do {
+               $m = $sth->fetchall_arrayref({}, 1000);
+               for my $x (@$m) {
+                       PublicInbox::Over::load_from_row($x);
+                       push(@cache, $x->{num}, $x->{bytes} + 0, $x->{blob});
+                       undef $x; # saves ~1.5M memory w/ 50k messages
+               }
+       } while (scalar(@$m) && ($beg = $cache[-3] + 1));
+       \@cache;
 }
 
 sub cmd_stat {