inbox: base_url method takes PSGI env hashref instead

[public-inbox.git] / lib / PublicInbox / SearchIdx.pm

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 67272997b22149faf3db8b9018a86ac8569f83ae..c2bf9a2ddc25d3232a58fb526566665407582ec6 100644 (file)
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -9,8 +9,12 @@
 package PublicInbox::SearchIdx;
 use strict;
 use warnings;
+use Email::MIME;
+use Email::MIME::ContentType;
+$Email::MIME::ContentType::STRICT_PARAMS = 0;
 use base qw(PublicInbox::Search);
-use PublicInbox::MID qw/mid_clean id_compress/;
+use PublicInbox::MID qw/mid_clean id_compress mid_mime/;
+use PublicInbox::MsgIter;
 require PublicInbox::Git;
 *xpfx = *PublicInbox::Search::xpfx;
 
@@ -54,7 +58,7 @@ sub add_message {
        my $db = $self->{xdb};
 
        my $doc_id;
-       my $mid = mid_clean($mime->header('Message-ID'));
+       my $mid = mid_clean(mid_mime($mime));
        my $was_ghost = 0;
        my $ct_msg = $mime->header('Content-Type') || 'text/plain';
 
@@ -114,9 +118,8 @@ sub add_message {
                $tg->index_text($smsg->from);
                $tg->increase_termpos;
 
-               $mime->walk_parts(sub {
-                       my ($part) = @_;
-                       return if $part->subparts; # walk_parts already recurses
+               msg_iter($mime, sub {
+                       my ($part, $depth, @idx) = @{$_[0]};
                        my $ct = $part->content_type || $ct_msg;
 
                        # account for filter bugs...
@@ -147,11 +150,11 @@ sub add_message {
 
                if ($was_ghost) {
                        $doc_id = $smsg->doc_id;
-                       $self->link_message($smsg, 0);
+                       $self->link_message($smsg, $smsg->thread_id);
                        $doc->set_data($smsg->to_doc_data);
                        $db->replace_document($doc_id, $doc);
                } else {
-                       $self->link_message($smsg, 0);
+                       $self->link_message($smsg);
                        $doc->set_data($smsg->to_doc_data);
                        $doc_id = $db->add_document($doc);
                }
@@ -208,23 +211,14 @@ sub next_thread_id {
 }
 
 sub link_message {
-       my ($self, $smsg, $is_ghost) = @_;
-
-       if ($is_ghost) {
-               $smsg->ensure_metadata;
-       } else {
-               $self->link_message_to_parents($smsg);
-       }
-}
-
-sub link_message_to_parents {
-       my ($self, $smsg) = @_;
+       my ($self, $smsg, $old_tid) = @_;
        my $doc = $smsg->{doc};
        my $mid = $smsg->mid;
        my $mime = $smsg->mime;
-       my $refs = $mime->header('References');
+       my $hdr = $mime->header_obj;
+       my $refs = $hdr->header_raw('References');
        my @refs = $refs ? ($refs =~ /<([^>]+)>/g) : ();
-       if (my $irt = $mime->header('In-Reply-To')) {
+       if (my $irt = $hdr->header_raw('In-Reply-To')) {
                # last References should be $irt
                # we will de-dupe later
                push @refs, mid_clean($irt);
@@ -253,6 +247,7 @@ sub link_message_to_parents {
                # but we can never trust clients to do the right thing
                my $ref = shift @refs;
                $tid = $self->_resolve_mid_to_tid($ref);
+               $self->merge_threads($tid, $old_tid) if defined $old_tid;
 
                # the rest of the refs should point to this tid:
                foreach $ref (@refs) {
@@ -274,29 +269,29 @@ sub index_blob {
 
 sub unindex_blob {
        my ($self, $git, $mime) = @_;
-       my $mid = mid_clean($mime->header('Message-ID'));
+       my $mid = eval { mid_clean(mid_mime($mime)) };
        $self->remove_message($mid) if defined $mid;
 }
 
 sub index_mm {
        my ($self, $git, $mime) = @_;
-       $self->{mm}->mid_insert(mid_clean($mime->header('Message-ID')));
+       $self->{mm}->mid_insert(mid_clean(mid_mime($mime)));
 }
 
 sub unindex_mm {
        my ($self, $git, $mime) = @_;
-       $self->{mm}->mid_delete(mid_clean($mime->header('Message-ID')));
+       $self->{mm}->mid_delete(mid_clean(mid_mime($mime)));
 }
 
 sub index_mm2 {
        my ($self, $git, $mime, $bytes) = @_;
-       my $num = $self->{mm}->num_for(mid_clean($mime->header('Message-ID')));
+       my $num = $self->{mm}->num_for(mid_clean(mid_mime($mime)));
        index_blob($self, $git, $mime, $bytes, $num);
 }
 
 sub unindex_mm2 {
        my ($self, $git, $mime) = @_;
-       $self->{mm}->mid_delete(mid_clean($mime->header('Message-ID')));
+       $self->{mm}->mid_delete(mid_clean(mid_mime($mime)));
        unindex_blob($self, $git, $mime);
 }
 
@@ -316,6 +311,8 @@ sub do_cat_mail {
        my ($git, $blob, $sizeref) = @_;
        my $mime = eval {
                my $str = $git->cat_file($blob, $sizeref);
+               # fixup bugs from import:
+               $$str =~ s/\A[\r\n]*From [^\r\n]*\r?\n//s;
                Email::MIME->new($str);
        };
        $@ ? undef : $mime;
@@ -337,7 +334,8 @@ sub rlog {
                                --raw -r --no-abbrev/, $range);
        my $latest;
        my $bytes;
-       while (my $line = <$log>) {
+       local $/ = "\n";
+       while (defined(my $line = <$log>)) {
                if ($line =~ /$addmsg/o) {
                        my $mime = do_cat_mail($git, $1, \$bytes) or next;
                        $add_cb->($self, $git, $mime, $bytes);
@@ -348,7 +346,6 @@ sub rlog {
                        $latest = $1;
                }
        }
-       close $log;
        $latest;
 }
 
@@ -410,17 +407,15 @@ sub _resolve_mid_to_tid {
 }
 
 sub create_ghost {
-       my ($self, $mid, $tid) = @_;
-
-       $tid = $self->next_thread_id unless defined $tid;
+       my ($self, $mid) = @_;
 
+       my $tid = $self->next_thread_id;
        my $doc = Search::Xapian::Document->new;
        $doc->add_term(xpfx('mid') . $mid);
        $doc->add_term(xpfx('thread') . $tid);
        $doc->add_term(xpfx('type') . 'ghost');
 
        my $smsg = PublicInbox::SearchMsg->wrap($doc, $mid);
-       $self->link_message($smsg, 1);
        $self->{xdb}->add_document($doc);
 
        $smsg;
@@ -445,8 +440,8 @@ sub _read_git_config_perm {
        my ($self) = @_;
        my @cmd = qw(config core.sharedRepository);
        my $fh = PublicInbox::Git->new($self->{git_dir})->popen(@cmd);
+       local $/ = "\n";
        my $perm = <$fh>;
-       close $fh;
        chomp $perm if defined $perm;
        $perm;
 }