isearch: emulate per-inbox search with ->ALL

[public-inbox.git] / lib / PublicInbox / Mbox.pm

diff --git a/lib/PublicInbox/Mbox.pm b/lib/PublicInbox/Mbox.pm
index abdf43c93ac7c8059750a08f02d71495bd7fb53e..1945915059905946d94ce20cca2f192f7845aef7 100644 (file)
--- a/lib/PublicInbox/Mbox.pm
+++ b/lib/PublicInbox/Mbox.pm
@@ -1,12 +1,8 @@
 # Copyright (C) 2015-2020 all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
-# Streaming (via getline) interface for formatting messages as an mboxrd.
-# Used by the PSGI web interface.
-#
-# public-inbox-httpd favors "getline" response bodies to take a
-# "pull"-based approach to feeding slow clients (as opposed to a
-# more common "push" model)
+# Streaming interface for mboxrd HTTP responses
+# See PublicInbox::GzipFilter for details.
 package PublicInbox::Mbox;
 use strict;
 use parent 'PublicInbox::GzipFilter';
@@ -33,7 +29,7 @@ sub getline {
        }
 }
 
-# called by PublicInbox::DS::write
+# called by PublicInbox::DS::write after http->next_step
 sub async_next {
        my ($http) = @_; # PublicInbox::HTTP
        my $ctx = $http->{forward} or return; # client aborted
@@ -44,7 +40,7 @@ sub async_next {
        warn "E: $@" if $@;
 }
 
-sub async_eml { # ->{async_eml} for async_blob_cb
+sub async_eml { # for async_blob_cb
        my ($ctx, $eml) = @_;
        my $smsg = delete $ctx->{smsg};
        # next message
@@ -56,9 +52,9 @@ sub async_eml { # ->{async_eml} for async_blob_cb
 
 sub res_hdr ($$) {
        my ($ctx, $subject) = @_;
-       my $fn = $subject // 'no-subject';
+       my $fn = $subject // '';
        $fn =~ s/^re:\s+//i;
-       $fn = $fn eq '' ? 'no-subject' : to_filename($fn);
+       $fn = to_filename($fn) // 'no-subject';
        my @hdr = ('Content-Type');
        if ($ctx->{-inbox}->{obfuscate}) {
                # obfuscation is stupid, but maybe scrapers are, too...
@@ -91,7 +87,7 @@ sub emit_raw {
        my $smsg = $ctx->{smsg} = $over->next_by_mid(@$mip) or return;
        my $res_hdr = res_hdr($ctx, $smsg->{subject});
        bless $ctx, __PACKAGE__;
-       $ctx->psgi_response(200, $res_hdr, \&async_next, \&async_eml);
+       $ctx->psgi_response(200, $res_hdr);
 }
 
 sub msg_hdr ($$;$) {
@@ -171,13 +167,13 @@ sub thread_mbox {
 sub emit_range {
        my ($ctx, $range) = @_;
 
-       my $query;
+       my $q;
        if ($range eq 'all') { # TODO: YYYY[-MM]
-               $query = '';
+               $q = '';
        } else {
                return [404, [qw(Content-Type text/plain)], []];
        }
-       mbox_all($ctx, $query);
+       mbox_all($ctx, { q => $q });
 }
 
 sub all_ids_cb {
@@ -207,40 +203,78 @@ sub mbox_all_ids {
        PublicInbox::MboxGz::mbox_gz($ctx, \&all_ids_cb, 'all');
 }
 
+sub gone ($$) {
+       my ($ctx, $what) = @_;
+       warn "W: `$ctx->{-inbox}->{inboxdir}' $what went away unexpectedly\n";
+       undef;
+}
+
 sub results_cb {
        my ($ctx) = @_;
-       my $mset = $ctx->{mset};
-       my $srch = $ctx->{srch};
+       my $over = $ctx->{-inbox}->over or return gone($ctx, 'over');
        while (1) {
-               while (my $mi = (($mset->items)[$ctx->{iter}++])) {
-                       my $smsg = PublicInbox::Smsg::from_mitem($mi,
-                                                               $srch) or next;
+               while (defined(my $num = shift(@{$ctx->{ids}}))) {
+                       my $smsg = $over->get_art($num) or next;
                        return $smsg;
                }
                # refill result set
-               $mset = $ctx->{mset} = $srch->query($ctx->{query},
-                                                       $ctx->{qopts});
+               my $srch = $ctx->{-inbox}->isrch or return gone($ctx, 'search');
+               my $mset = $srch->mset($ctx->{query}, $ctx->{qopts});
                my $size = $mset->size or return;
                $ctx->{qopts}->{offset} += $size;
-               $ctx->{iter} = 0;
+               $ctx->{ids} = $srch->mset_to_artnums($mset);
        }
 }
 
+sub results_thread_cb {
+       my ($ctx) = @_;
+
+       my $over = $ctx->{-inbox}->over or return gone($ctx, 'over');
+       while (1) {
+               while (defined(my $num = shift(@{$ctx->{xids}}))) {
+                       my $smsg = $over->get_art($num) or next;
+                       return $smsg;
+               }
+
+               # refills ctx->{xids}
+               next if $over->expand_thread($ctx);
+
+               # refill result set
+               my $srch = $ctx->{-inbox}->isrch or return gone($ctx, 'search');
+               my $mset = $srch->mset($ctx->{query}, $ctx->{qopts});
+               my $size = $mset->size or return;
+               $ctx->{qopts}->{offset} += $size;
+               $ctx->{ids} = $srch->mset_to_artnums($mset);
+       }
+
+}
+
 sub mbox_all {
-       my ($ctx, $query) = @_;
+       my ($ctx, $q) = @_;
+       my $q_string = $q->{'q'};
+       return mbox_all_ids($ctx) if $q_string !~ /\S/;
+       my $srch = $ctx->{-inbox}->isrch or
+               return PublicInbox::WWW::need($ctx, 'Search');
+       my $over = $ctx->{-inbox}->over or
+               return PublicInbox::WWW::need($ctx, 'Overview');
 
-       return mbox_all_ids($ctx) if $query eq '';
-       my $qopts = $ctx->{qopts} = { mset => 2 };
-       my $srch = $ctx->{srch} = $ctx->{-inbox}->search or
-               return PublicInbox::WWW::need($ctx, 'Search');;
-       my $mset = $ctx->{mset} = $srch->query($query, $qopts);
+       my $qopts = $ctx->{qopts} = { mset => 2 }; # order by docid
+       $qopts->{thread} = 1 if $q->{t};
+       my $mset = $srch->mset($q_string, $qopts);
        $qopts->{offset} = $mset->size or
                        return [404, [qw(Content-Type text/plain)],
                                ["No results found\n"]];
-       $ctx->{iter} = 0;
-       $ctx->{query} = $query;
+       $ctx->{query} = $q_string;
+       $ctx->{ids} = $srch->mset_to_artnums($mset);
        require PublicInbox::MboxGz;
-       PublicInbox::MboxGz::mbox_gz($ctx, \&results_cb, 'results-'.$query);
+       my $fn;
+       if ($q->{t} && $srch->has_threadid) {
+               $fn = 'results-thread-'.$q_string;
+               PublicInbox::MboxGz::mbox_gz($ctx, \&results_thread_cb, $fn);
+       } else {
+               $fn = 'results-'.$q_string;
+               PublicInbox::MboxGz::mbox_gz($ctx, \&results_cb, $fn);
+       }
 }
 
 1;