www: label sections and hopefully improve navigation

[public-inbox.git] / lib / PublicInbox / SearchIdx.pm

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 63be68101b40c8ae7f4f8d8e39ee94492594a589..c2bf9a2ddc25d3232a58fb526566665407582ec6 100644 (file)
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -9,8 +9,12 @@
 package PublicInbox::SearchIdx;
 use strict;
 use warnings;
+use Email::MIME;
+use Email::MIME::ContentType;
+$Email::MIME::ContentType::STRICT_PARAMS = 0;
 use base qw(PublicInbox::Search);
 use PublicInbox::MID qw/mid_clean id_compress mid_mime/;
+use PublicInbox::MsgIter;
 require PublicInbox::Git;
 *xpfx = *PublicInbox::Search::xpfx;
 
@@ -114,9 +118,8 @@ sub add_message {
                $tg->index_text($smsg->from);
                $tg->increase_termpos;
 
-               $mime->walk_parts(sub {
-                       my ($part) = @_;
-                       return if $part->subparts; # walk_parts already recurses
+               msg_iter($mime, sub {
+                       my ($part, $depth, @idx) = @{$_[0]};
                        my $ct = $part->content_type || $ct_msg;
 
                        # account for filter bugs...
@@ -147,11 +150,11 @@ sub add_message {
 
                if ($was_ghost) {
                        $doc_id = $smsg->doc_id;
-                       $self->link_message($smsg, 0);
+                       $self->link_message($smsg, $smsg->thread_id);
                        $doc->set_data($smsg->to_doc_data);
                        $db->replace_document($doc_id, $doc);
                } else {
-                       $self->link_message($smsg, 0);
+                       $self->link_message($smsg);
                        $doc->set_data($smsg->to_doc_data);
                        $doc_id = $db->add_document($doc);
                }
@@ -208,17 +211,7 @@ sub next_thread_id {
 }
 
 sub link_message {
-       my ($self, $smsg, $is_ghost) = @_;
-
-       if ($is_ghost) {
-               $smsg->ensure_metadata;
-       } else {
-               $self->link_message_to_parents($smsg);
-       }
-}
-
-sub link_message_to_parents {
-       my ($self, $smsg) = @_;
+       my ($self, $smsg, $old_tid) = @_;
        my $doc = $smsg->{doc};
        my $mid = $smsg->mid;
        my $mime = $smsg->mime;
@@ -254,6 +247,7 @@ sub link_message_to_parents {
                # but we can never trust clients to do the right thing
                my $ref = shift @refs;
                $tid = $self->_resolve_mid_to_tid($ref);
+               $self->merge_threads($tid, $old_tid) if defined $old_tid;
 
                # the rest of the refs should point to this tid:
                foreach $ref (@refs) {
@@ -317,6 +311,8 @@ sub do_cat_mail {
        my ($git, $blob, $sizeref) = @_;
        my $mime = eval {
                my $str = $git->cat_file($blob, $sizeref);
+               # fixup bugs from import:
+               $$str =~ s/\A[\r\n]*From [^\r\n]*\r?\n//s;
                Email::MIME->new($str);
        };
        $@ ? undef : $mime;
@@ -338,6 +334,7 @@ sub rlog {
                                --raw -r --no-abbrev/, $range);
        my $latest;
        my $bytes;
+       local $/ = "\n";
        while (defined(my $line = <$log>)) {
                if ($line =~ /$addmsg/o) {
                        my $mime = do_cat_mail($git, $1, \$bytes) or next;
@@ -410,17 +407,15 @@ sub _resolve_mid_to_tid {
 }
 
 sub create_ghost {
-       my ($self, $mid, $tid) = @_;
-
-       $tid = $self->next_thread_id unless defined $tid;
+       my ($self, $mid) = @_;
 
+       my $tid = $self->next_thread_id;
        my $doc = Search::Xapian::Document->new;
        $doc->add_term(xpfx('mid') . $mid);
        $doc->add_term(xpfx('thread') . $tid);
        $doc->add_term(xpfx('type') . 'ghost');
 
        my $smsg = PublicInbox::SearchMsg->wrap($doc, $mid);
-       $self->link_message($smsg, 1);
        $self->{xdb}->add_document($doc);
 
        $smsg;
@@ -445,6 +440,7 @@ sub _read_git_config_perm {
        my ($self) = @_;
        my @cmd = qw(config core.sharedRepository);
        my $fh = PublicInbox::Git->new($self->{git_dir})->popen(@cmd);
+       local $/ = "\n";
        my $perm = <$fh>;
        chomp $perm if defined $perm;
        $perm;