lib/PublicInbox/Inbox.pm | 12 ++++++++++++ lib/PublicInbox/Mbox.pm | 2 +- lib/PublicInbox/SearchIdx.pm | 24 +++++++++++++----------- lib/PublicInbox/SearchMsg.pm | 20 ++++++++++++++++---- diff --git a/lib/PublicInbox/Inbox.pm b/lib/PublicInbox/Inbox.pm index 4fbbb522f9c9d208dd54a240a3b68662f90163a5..e552cd4f2ff38696bee4d2370f5b33aa10011561 100644 --- a/lib/PublicInbox/Inbox.pm +++ b/lib/PublicInbox/Inbox.pm @@ -162,6 +162,18 @@ $$str =~ s/\A[\r\n]*From [^\r\n]*\r?\n//s if $str; $str; } +sub msg_by_smsg ($$;$) { + my ($self, $smsg, $ref) = @_; + + # backwards compat to fallback to msg_by_mid + # TODO: remove if we bump SCHEMA_VERSION in Search.pm: + defined(my $blob = $smsg->blob) or return msg_by_mid($self, $smsg->mid); + + my $str = git($self)->cat_file($blob, $ref); + $$str =~ s/\A[\r\n]*From [^\r\n]*\r?\n//s if $str; + $str; +} + sub path_check { my ($self, $path) = @_; git($self)->check('HEAD:'.$path); diff --git a/lib/PublicInbox/Mbox.pm b/lib/PublicInbox/Mbox.pm index d2c095445778f368160a165f6b14cf8921e1bb43..1e3de5b0d484550b940ce977ff27f8d015ed757e 100644 --- a/lib/PublicInbox/Mbox.pm +++ b/lib/PublicInbox/Mbox.pm @@ -129,7 +129,7 @@ my $ibx = $ctx->{-inbox}; my $gz = $self->{gz}; do { while (defined(my $smsg = shift @{$self->{msgs}})) { - my $msg = eval { $ibx->msg_by_mid($smsg->mid) } or next; + my $msg = eval { $ibx->msg_by_smsg($smsg) } or next; $msg = Email::Simple->new($msg); $gz->write(PublicInbox::Mbox::msg_str($ctx, $msg)); my $bref = $self->{buf}; diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm index f0a3687d1aecf146319f1605faebed1247348268..f8249c50d4621ed0d3d172d7e992987898609ef0 100644 --- a/lib/PublicInbox/SearchIdx.pm +++ b/lib/PublicInbox/SearchIdx.pm @@ -91,7 +91,7 @@ $doc->add_value($col, $num); } sub add_message { - my ($self, $mime, $bytes, $num) = @_; # mime = Email::MIME object + my ($self, $mime, $bytes, $num, $blob) = @_; # mime = Email::MIME object my $db = $self->{xdb}; my ($doc_id, $old_tid); @@ -170,7 +170,7 @@ } }); link_message($self, $smsg, $old_tid); - $doc->set_data($smsg->to_doc_data); + $doc->set_data($smsg->to_doc_data($blob)); if (defined $doc_id) { $db->replace_document($doc_id, $doc); } else { @@ -279,8 +279,8 @@ $doc->add_term(xpfx('thread') . $tid); } sub index_blob { - my ($self, $git, $mime, $bytes, $num) = @_; - $self->add_message($mime, $bytes, $num); + my ($self, $git, $mime, $bytes, $num, $blob) = @_; + $self->add_message($mime, $bytes, $num, $blob); } sub unindex_blob { @@ -300,9 +300,9 @@ $self->{mm}->mid_delete(mid_clean(mid_mime($mime))); } sub index_mm2 { - my ($self, $git, $mime, $bytes) = @_; + my ($self, $git, $mime, $bytes, $blob) = @_; my $num = $self->{mm}->num_for(mid_clean(mid_mime($mime))); - index_blob($self, $git, $mime, $bytes, $num); + index_blob($self, $git, $mime, $bytes, $num, $blob); } sub unindex_mm2 { @@ -312,9 +312,9 @@ unindex_blob($self, $git, $mime); } sub index_both { - my ($self, $git, $mime, $bytes) = @_; + my ($self, $git, $mime, $bytes, $blob) = @_; my $num = index_mm($self, $git, $mime); - index_blob($self, $git, $mime, $bytes, $num); + index_blob($self, $git, $mime, $bytes, $num, $blob); } sub unindex_both { @@ -355,10 +355,12 @@ local $/ = "\n"; my $line; while (defined($line = <$log>)) { if ($line =~ /$addmsg/o) { - my $mime = do_cat_mail($git, $1, \$bytes) or next; - $add_cb->($self, $git, $mime, $bytes); + my $blob = $1; + my $mime = do_cat_mail($git, $blob, \$bytes) or next; + $add_cb->($self, $git, $mime, $bytes, $blob); } elsif ($line =~ /$delmsg/o) { - my $mime = do_cat_mail($git, $1) or next; + my $blob = $1; + my $mime = do_cat_mail($git, $blob) or next; $del_cb->($self, $git, $mime); } elsif ($line =~ /^commit ($h40)/o) { if (defined $max && --$max <= 0) { diff --git a/lib/PublicInbox/SearchMsg.pm b/lib/PublicInbox/SearchMsg.pm index 4b0b645e17af961b9bf450f57756fce97acba481..9d873c4a1899a6d9c6f0364e5ad19df2dd88748d 100644 --- a/lib/PublicInbox/SearchMsg.pm +++ b/lib/PublicInbox/SearchMsg.pm @@ -38,7 +38,7 @@ my ($class, $doc) = @_; my $data = $doc->get_data or return; my $ts = get_val($doc, &PublicInbox::Search::TS); utf8::decode($data); - my ($subj, $from, $refs, $to, $cc) = split(/\n/, $data); + my ($subj, $from, $refs, $to, $cc, $blob) = split(/\n/, $data); bless { doc => $doc, subject => $subj, @@ -47,6 +47,7 @@ from => $from, references => $refs, to => $to, cc => $cc, + blob => $blob, }, $class; } @@ -105,9 +106,11 @@ $self->{ts} ||= eval { str2time($self->mime->header('Date')) } || 0; } sub to_doc_data { - my ($self) = @_; - join("\n", $self->subject, $self->from, $self->references, - $self->to, $self->cc); + my ($self, $blob) = @_; + my @rows = ($self->subject, $self->from, $self->references, + $self->to, $self->cc); + push @rows, $blob if defined $blob; + join("\n", @rows); } sub references { @@ -184,6 +187,15 @@ } } sub _extract_mid { mid_clean(mid_mime($_[0]->mime)) } + +sub blob { + my ($self, $x40) = @_; + if (defined $x40) { + $self->{blob} = $x40; + } else { + $self->{blob}; + } +} sub mime { my ($self, $mime) = @_;