mbox: remove remaining OFFSET usage in SQLite

[public-inbox.git] / lib / PublicInbox / Mbox.pm

diff --git a/lib/PublicInbox/Mbox.pm b/lib/PublicInbox/Mbox.pm
index 84cc384529f174cd020bbfb2360519720b5d0929..0be1968584d9c68a7a76a72b98f82afea13bc564 100644 (file)
--- a/lib/PublicInbox/Mbox.pm
+++ b/lib/PublicInbox/Mbox.pm
@@ -26,12 +26,60 @@ sub subject_fn ($) {
        $fn eq '' ? 'no-subject' : $fn;
 }
 
-sub emit1 {
-       my ($ctx, $msg) = @_;
-       $msg = Email::Simple->new($msg);
-       my $fn = subject_fn($msg);
+sub mb_stream {
+       my ($more) = @_;
+       bless $more, 'PublicInbox::Mbox';
+}
+
+# called by PSGI server as body response
+sub getline {
+       my ($more) = @_; # self
+       my ($ctx, $head, $tail, $db, $cur) = @$more;
+       if ($cur) {
+               pop @$more;
+               return msg_str($ctx, $cur);
+       }
+       for (; !defined($cur) && $head != $tail; $head++) {
+               my $smsg = PublicInbox::SearchMsg->get($head, $db, $ctx->{mid});
+               my $mref = $ctx->{-inbox}->msg_by_smsg($smsg) or next;
+               $cur = Email::Simple->new($mref);
+               $cur = msg_str($ctx, $cur);
+       }
+       $more->[1] = $head;
+       $cur;
+}
+
+sub close {} # noop
+
+sub emit_raw {
+       my ($ctx) = @_;
+       my $mid = $ctx->{mid};
+       my $ibx = $ctx->{-inbox};
+       my $first;
+       my $more;
+       my ($head, $tail, $db);
+       my %seen;
+       if (my $srch = $ibx->search) {
+               $srch->retry_reopen(sub {
+                       ($head, $tail, $db) = $srch->each_smsg_by_mid($mid);
+                       for (; !defined($first) && $head != $tail; $head++) {
+                               my @args = ($head, $db, $mid);
+                               my $smsg = PublicInbox::SearchMsg->get(@args);
+                               my $mref = $ibx->msg_by_smsg($smsg) or next;
+                               $first = Email::Simple->new($mref);
+                       }
+                       if ($head != $tail) {
+                               $more = [ $ctx, $head, $tail, $db, $first ];
+                       }
+               });
+       } else {
+               my $mref = $ibx->msg_by_mid($mid) or return;
+               $first = Email::Simple->new($mref);
+       }
+       return unless defined $first;
+       my $fn = subject_fn($first);
        my @hdr = ('Content-Type');
-       if ($ctx->{-inbox}->{obfuscate}) {
+       if ($ibx->{obfuscate}) {
                # obfuscation is stupid, but maybe scrapers are, too...
                push @hdr, 'application/mbox';
                $fn .= '.mbox';
@@ -40,14 +88,11 @@ sub emit1 {
                $fn .= '.txt';
        }
        push @hdr, 'Content-Disposition', "inline; filename=$fn";
-
-       # single message should be easily renderable in browsers,
-       # unless obfuscation is enabled :<
-       [ 200, \@hdr, [ msg_str($ctx, $msg) ] ]
+       [ 200, \@hdr, $more ? mb_stream($more) : [ msg_str($ctx, $first) ] ];
 }
 
 sub msg_str {
-       my ($ctx, $simple) = @_; # Email::Simple object
+       my ($ctx, $simple, $mid) = @_; # Email::Simple object
        my $header_obj = $simple->header_obj;
 
        # drop potentially confusing headers, ssoma already should've dropped
@@ -57,7 +102,7 @@ sub msg_str {
        }
        my $ibx = $ctx->{-inbox};
        my $base = $ibx->base_url($ctx->{env});
-       my $mid = mid_clean($header_obj->header('Message-ID'));
+       $mid = $ctx->{mid} unless defined $mid;
        $mid = mid_escape($mid);
        my @append = (
                'Archived-At', "<$base$mid/>",
@@ -93,8 +138,12 @@ sub thread_mbox {
        my ($ctx, $srch, $sfx) = @_;
        eval { require IO::Compress::Gzip };
        return sub { need_gzip(@_) } if $@;
-
-       my $cb = sub { $srch->get_thread($ctx->{mid}, @_) };
+       my $prev = 0;
+       my $cb = sub {
+               my $msgs = $srch->get_thread($ctx->{mid}, $prev);
+               $prev = $msgs->[-1]->{num} if scalar(@$msgs);
+               $msgs;
+       };
        PublicInbox::MboxGz->response($ctx, $cb);
 }
 
@@ -115,7 +164,25 @@ sub mbox_all {
 
        eval { require IO::Compress::Gzip };
        return sub { need_gzip(@_) } if $@;
-       my $cb = sub { $ctx->{srch}->query($query, @_) };
+       if ($query eq '') {
+               my $prev = 0;
+               my $msgs = [];
+               my $cb = sub {
+                       $ctx->{-inbox}->mm->id_batch($prev, sub {
+                               $msgs = $_[0];
+                       });
+                       $prev = $msgs->[-1] if @$msgs;
+                       $msgs;
+               };
+               return PublicInbox::MboxGz->response($ctx, $cb, 'all');
+       }
+       my $opts = { offset => 0 };
+       my $srch = $ctx->{srch};
+       my $cb = sub { # called by MboxGz->getline
+               my $msgs = $srch->query($query, $opts);
+               $opts->{offset} += scalar @$msgs;
+               $msgs;
+       };
        PublicInbox::MboxGz->response($ctx, $cb, 'results-'.$query);
 }
 
@@ -147,7 +214,6 @@ sub new {
                cb => $cb,
                ctx => $ctx,
                msgs => [],
-               opts => { offset => 0 },
        }, $class;
 }
 
@@ -172,15 +238,20 @@ sub set_filename ($$) {
 sub getline {
        my ($self) = @_;
        my $ctx = $self->{ctx} or return;
-       my $res;
        my $ibx = $ctx->{-inbox};
        my $gz = $self->{gz};
+       my $msgs = $self->{msgs};
        do {
                # work on existing result set
-               while (defined(my $smsg = shift @{$self->{msgs}})) {
+               while (defined(my $smsg = shift @$msgs)) {
+                       # id_batch may return integers
+                       ref($smsg) or
+                               $smsg = $ctx->{srch}->{over_ro}->get_art($smsg);
+
                        my $msg = eval { $ibx->msg_by_smsg($smsg) } or next;
                        $msg = Email::Simple->new($msg);
-                       $gz->write(PublicInbox::Mbox::msg_str($ctx, $msg));
+                       $gz->write(PublicInbox::Mbox::msg_str($ctx, $msg,
+                                                               $smsg->mid));
 
                        # use subject of first message as subject
                        if (my $hdr = delete $self->{hdr}) {
@@ -201,12 +272,10 @@ sub getline {
                }
 
                # refill result set
-               $res = $self->{cb}->($self->{opts});
-               $self->{msgs} = $res->{msgs};
-               $res = scalar @{$self->{msgs}};
-               $self->{opts}->{offset} += $res;
-       } while ($res);
+               $msgs = $self->{msgs} = $self->{cb}->();
+       } while (@$msgs);
        $gz->close;
+       # signal that we're done and can return undef next call:
        delete $self->{ctx};
        ${delete $self->{buf}};
 }