- return if $ct =~ m!\btext/x?html\b!i;
-
- my $s = eval { $part->body_str };
- if ($@) {
- if ($ct =~ m!\btext/plain\b!i) {
- # Try to assume UTF-8 because Alpine
- # seems to do wacky things and set
- # charset=X-UNKNOWN
- $part->charset_set('UTF-8');
- $s = eval { $part->body_str };
- $s = $part->body if $@;
- }
- }
- defined $s or return;
-
- my (@orig, @quot);
- my $body = $part->body;
- my @lines = split(/\n/, $body);
- while (defined(my $l = shift @lines)) {
- if ($l =~ /^>/) {
- index_body($tg, \@orig, $doc) if @orig;
- push @quot, $l;
- } else {
- index_body($tg, \@quot, 0) if @quot;
- push @orig, $l;
- }
- }
- index_body($tg, \@quot, 0) if @quot;
- index_body($tg, \@orig, $doc) if @orig;
- });
-
- # populates smsg->references for smsg->to_doc_data
- my $data = $smsg->to_doc_data($oid, $mid0);
- foreach my $mid (@$mids) {
- $tg->index_text($mid, 1, 'XM');
+ my ($s, undef) = msg_part_text($part, $ct);
+ defined $s or return;
+
+ # split off quoted and unquoted blocks:
+ my @sections = split(/((?:^>[^\n]*\n)+)/sm, $s);
+ $part = $s = undef;
+ index_body($self, $_, /\A>/ ? 0 : $doc) for @sections;
+}
+
+sub add_xapian ($$$$) {
+ my ($self, $mime, $smsg, $mids) = @_;
+ $smsg->{mime} = $mime; # XXX dangerous
+ my $hdr = $mime->header_obj;
+ my $doc = $X->{Document}->new;
+ my $subj = $smsg->subject;
+ add_val($doc, PublicInbox::Search::TS(), $smsg->{ts});
+ my @ds = gmtime($smsg->{ds});
+ my $yyyymmdd = strftime('%Y%m%d', @ds);
+ add_val($doc, PublicInbox::Search::YYYYMMDD(), $yyyymmdd);
+ my $dt = strftime('%Y%m%d%H%M%S', @ds);
+ add_val($doc, PublicInbox::Search::DT(), $dt);
+
+ my $tg = term_generator($self);
+
+ $tg->set_document($doc);
+ index_text($self, $subj, 1, 'S') if $subj;
+ index_users($self, $smsg);
+
+ msg_iter($mime, \&index_xapian, [ $self, $doc ]);
+ foreach my $mid (@$mids) {
+ index_text($self, $mid, 1, 'XM');
+
+ # because too many Message-IDs are prefixed with
+ # "Pine.LNX."...
+ if ($mid =~ /\w{12,}/) {
+ my @long = ($mid =~ /(\w{3,}+)/g);
+ index_text($self, join(' ', @long), 1, 'XM');