www: use WwwStream for dumping thread and search views

[public-inbox.git] / lib / PublicInbox / SearchIdx.pm

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 4a4b2bdbc67201a84aea9343b72f1b5fc3f73175..c2bf9a2ddc25d3232a58fb526566665407582ec6 100644 (file)
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -9,6 +9,9 @@
 package PublicInbox::SearchIdx;
 use strict;
 use warnings;
+use Email::MIME;
+use Email::MIME::ContentType;
+$Email::MIME::ContentType::STRICT_PARAMS = 0;
 use base qw(PublicInbox::Search);
 use PublicInbox::MID qw/mid_clean id_compress mid_mime/;
 use PublicInbox::MsgIter;
@@ -147,11 +150,11 @@ sub add_message {
 
                if ($was_ghost) {
                        $doc_id = $smsg->doc_id;
-                       $self->link_message($smsg, 0);
+                       $self->link_message($smsg, $smsg->thread_id);
                        $doc->set_data($smsg->to_doc_data);
                        $db->replace_document($doc_id, $doc);
                } else {
-                       $self->link_message($smsg, 0);
+                       $self->link_message($smsg);
                        $doc->set_data($smsg->to_doc_data);
                        $doc_id = $db->add_document($doc);
                }
@@ -208,17 +211,7 @@ sub next_thread_id {
 }
 
 sub link_message {
-       my ($self, $smsg, $is_ghost) = @_;
-
-       if ($is_ghost) {
-               $smsg->ensure_metadata;
-       } else {
-               $self->link_message_to_parents($smsg);
-       }
-}
-
-sub link_message_to_parents {
-       my ($self, $smsg) = @_;
+       my ($self, $smsg, $old_tid) = @_;
        my $doc = $smsg->{doc};
        my $mid = $smsg->mid;
        my $mime = $smsg->mime;
@@ -254,6 +247,7 @@ sub link_message_to_parents {
                # but we can never trust clients to do the right thing
                my $ref = shift @refs;
                $tid = $self->_resolve_mid_to_tid($ref);
+               $self->merge_threads($tid, $old_tid) if defined $old_tid;
 
                # the rest of the refs should point to this tid:
                foreach $ref (@refs) {
@@ -317,6 +311,8 @@ sub do_cat_mail {
        my ($git, $blob, $sizeref) = @_;
        my $mime = eval {
                my $str = $git->cat_file($blob, $sizeref);
+               # fixup bugs from import:
+               $$str =~ s/\A[\r\n]*From [^\r\n]*\r?\n//s;
                Email::MIME->new($str);
        };
        $@ ? undef : $mime;
@@ -411,17 +407,15 @@ sub _resolve_mid_to_tid {
 }
 
 sub create_ghost {
-       my ($self, $mid, $tid) = @_;
-
-       $tid = $self->next_thread_id unless defined $tid;
+       my ($self, $mid) = @_;
 
+       my $tid = $self->next_thread_id;
        my $doc = Search::Xapian::Document->new;
        $doc->add_term(xpfx('mid') . $mid);
        $doc->add_term(xpfx('thread') . $tid);
        $doc->add_term(xpfx('type') . 'ghost');
 
        my $smsg = PublicInbox::SearchMsg->wrap($doc, $mid);
-       $self->link_message($smsg, 1);
        $self->{xdb}->add_document($doc);
 
        $smsg;