mbox: stream entire thread, regardless of size

Since mbox is usually downloaded, support fetching infinitely large
responses via streaming.

diff --git a/lib/PublicInbox/Mbox.pm b/lib/PublicInbox/Mbox.pm
index 2ec50656e36e7f02cd093db88168270a65d4bdff..fc9df1adcbc8b6f1c5ffb1393e61552faea6d714 100644 (file)
--- a/lib/PublicInbox/Mbox.pm
+++ b/lib/PublicInbox/Mbox.pm
@@ -9,27 +9,14 @@ use Fcntl qw(SEEK_SET);
 
 sub thread_mbox {
        my ($ctx, $srch) = @_;
-       my $mid = mid_compressed($ctx->{mid});
-       my $res = $srch->get_thread($mid);
-       my $msgs = delete $res->{msgs};
-       require PublicInbox::GitCatFile;
-       require Email::Simple;
-       my $git = PublicInbox::GitCatFile->new($ctx->{git_dir});
-
        sub {
-               my ($res) = @_; # Plack callback
-               my $w = $res->([200, [ 'Content-Type' => 'text/plain' ] ]);
-               while (defined(my $smsg = shift @$msgs)) {
-                       my $msg = eval {
-                               my $path = 'HEAD:' . mid2path($smsg->mid);
-                               Email::Simple->new($git->cat_file($path));
-                       };
-                       emit($w, $msg) if $msg;
-               }
+               my ($response) = @_; # Plack callback
+               my $w = $response->([200, ['Content-Type' => 'text/plain']]);
+               emit_mbox($w, $ctx, $srch);
        }
 }
 
-sub emit {
+sub emit_msg {
        my ($fh, $simple) = @_; # Email::Simple object
 
        # drop potentially confusing headers, ssoma already should've dropped
@@ -52,4 +39,29 @@ sub emit {
        $fh->write($buf);
 }
 
+sub emit_mbox {
+       my ($fh, $ctx, $srch) = @_;
+
+       require PublicInbox::GitCatFile;
+       require Email::Simple;
+       my $mid = mid_compressed($ctx->{mid});
+       my $git = PublicInbox::GitCatFile->new($ctx->{git_dir});
+       my %opts = (offset => 0);
+       my $nr;
+       do {
+               my $res = $srch->get_thread($mid, \%opts);
+               my $msgs = $res->{msgs};
+               $nr = scalar @$msgs;
+               while (defined(my $smsg = shift @$msgs)) {
+                       my $msg = eval {
+                               my $p = 'HEAD:'.mid2path($smsg->mid);
+                               Email::Simple->new($git->cat_file($p));
+                       };
+                       emit_msg($fh, $msg) if $msg;
+               }
+
+               $opts{offset} += $nr;
+       } while ($nr > 0);
+}
+
 1;

diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm
index f0040500ccefbd2abd56831a4b3c9ec5a6795c9f..f1ff3a4b79fa6ea8c05f0c3a104c69424aa05c68 100644 (file)
--- a/lib/PublicInbox/Search.pm
+++ b/lib/PublicInbox/Search.pm
@@ -247,7 +247,7 @@ sub get_thread {
        my $qtid = $qp->parse_query('thread:'.$smsg->thread_id);
        my $qsub = $qp->parse_query('path:'.mid_compressed($smsg->path));
        my $query = Search::Xapian::Query->new(OP_OR, $qtid, $qsub);
-       $self->do_enquire($query);
+       $self->do_enquire($query, $opts);
 }
 
 # private subs below