searchmsg: add git object ID to doc_data

Doing git tree lookups based on the SHA-1 of the Message-ID
is expensive as trees get larger, instead, use the SHA-1
object ID directly.  This drastically reduces the amount
of time spent in the "git cat-file --batch" process for
fetching the /$INBOX/all.mbox.gz endpoint on the ~800MB
git@vger.kernel.org mirror

This retains backwards compatibility and allows existing
indices to be transparently upgraded without performance
degradation.

diff --git a/lib/PublicInbox/Inbox.pm b/lib/PublicInbox/Inbox.pm
index 4fbbb522f9c9d208dd54a240a3b68662f90163a5..e552cd4f2ff38696bee4d2370f5b33aa10011561 100644 (file)
--- a/lib/PublicInbox/Inbox.pm
+++ b/lib/PublicInbox/Inbox.pm
@@ -162,6 +162,18 @@ sub msg_by_path ($$;$) {
        $str;
 }
 
+sub msg_by_smsg ($$;$) {
+       my ($self, $smsg, $ref) = @_;
+
+       # backwards compat to fallback to msg_by_mid
+       # TODO: remove if we bump SCHEMA_VERSION in Search.pm:
+       defined(my $blob = $smsg->blob) or return msg_by_mid($self, $smsg->mid);
+
+       my $str = git($self)->cat_file($blob, $ref);
+       $$str =~ s/\A[\r\n]*From [^\r\n]*\r?\n//s if $str;
+       $str;
+}
+
 sub path_check {
        my ($self, $path) = @_;
        git($self)->check('HEAD:'.$path);

diff --git a/lib/PublicInbox/Mbox.pm b/lib/PublicInbox/Mbox.pm
index d2c095445778f368160a165f6b14cf8921e1bb43..1e3de5b0d484550b940ce977ff27f8d015ed757e 100644 (file)
--- a/lib/PublicInbox/Mbox.pm
+++ b/lib/PublicInbox/Mbox.pm
@@ -129,7 +129,7 @@ sub getline {
        my $gz = $self->{gz};
        do {
                while (defined(my $smsg = shift @{$self->{msgs}})) {
-                       my $msg = eval { $ibx->msg_by_mid($smsg->mid) } or next;
+                       my $msg = eval { $ibx->msg_by_smsg($smsg) } or next;
                        $msg = Email::Simple->new($msg);
                        $gz->write(PublicInbox::Mbox::msg_str($ctx, $msg));
                        my $bref = $self->{buf};

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index f0a3687d1aecf146319f1605faebed1247348268..f8249c50d4621ed0d3d172d7e992987898609ef0 100644 (file)
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -91,7 +91,7 @@ sub add_val {
 }
 
 sub add_message {
-       my ($self, $mime, $bytes, $num) = @_; # mime = Email::MIME object
+       my ($self, $mime, $bytes, $num, $blob) = @_; # mime = Email::MIME object
        my $db = $self->{xdb};
 
        my ($doc_id, $old_tid);
@@ -170,7 +170,7 @@ sub add_message {
                });
 
                link_message($self, $smsg, $old_tid);
-               $doc->set_data($smsg->to_doc_data);
+               $doc->set_data($smsg->to_doc_data($blob));
                if (defined $doc_id) {
                        $db->replace_document($doc_id, $doc);
                } else {
@@ -279,8 +279,8 @@ sub link_message {
 }
 
 sub index_blob {
-       my ($self, $git, $mime, $bytes, $num) = @_;
-       $self->add_message($mime, $bytes, $num);
+       my ($self, $git, $mime, $bytes, $num, $blob) = @_;
+       $self->add_message($mime, $bytes, $num, $blob);
 }
 
 sub unindex_blob {
@@ -300,9 +300,9 @@ sub unindex_mm {
 }
 
 sub index_mm2 {
-       my ($self, $git, $mime, $bytes) = @_;
+       my ($self, $git, $mime, $bytes, $blob) = @_;
        my $num = $self->{mm}->num_for(mid_clean(mid_mime($mime)));
-       index_blob($self, $git, $mime, $bytes, $num);
+       index_blob($self, $git, $mime, $bytes, $num, $blob);
 }
 
 sub unindex_mm2 {
@@ -312,9 +312,9 @@ sub unindex_mm2 {
 }
 
 sub index_both {
-       my ($self, $git, $mime, $bytes) = @_;
+       my ($self, $git, $mime, $bytes, $blob) = @_;
        my $num = index_mm($self, $git, $mime);
-       index_blob($self, $git, $mime, $bytes, $num);
+       index_blob($self, $git, $mime, $bytes, $num, $blob);
 }
 
 sub unindex_both {
@@ -355,10 +355,12 @@ sub rlog {
        my $line;
        while (defined($line = <$log>)) {
                if ($line =~ /$addmsg/o) {
-                       my $mime = do_cat_mail($git, $1, \$bytes) or next;
-                       $add_cb->($self, $git, $mime, $bytes);
+                       my $blob = $1;
+                       my $mime = do_cat_mail($git, $blob, \$bytes) or next;
+                       $add_cb->($self, $git, $mime, $bytes, $blob);
                } elsif ($line =~ /$delmsg/o) {
-                       my $mime = do_cat_mail($git, $1) or next;
+                       my $blob = $1;
+                       my $mime = do_cat_mail($git, $blob) or next;
                        $del_cb->($self, $git, $mime);
                } elsif ($line =~ /^commit ($h40)/o) {
                        if (defined $max && --$max <= 0) {

diff --git a/lib/PublicInbox/SearchMsg.pm b/lib/PublicInbox/SearchMsg.pm
index 4b0b645e17af961b9bf450f57756fce97acba481..9d873c4a1899a6d9c6f0364e5ad19df2dd88748d 100644 (file)
--- a/lib/PublicInbox/SearchMsg.pm
+++ b/lib/PublicInbox/SearchMsg.pm
@@ -38,7 +38,7 @@ sub load_doc {
        my $data = $doc->get_data or return;
        my $ts = get_val($doc, &PublicInbox::Search::TS);
        utf8::decode($data);
-       my ($subj, $from, $refs, $to, $cc) = split(/\n/, $data);
+       my ($subj, $from, $refs, $to, $cc, $blob) = split(/\n/, $data);
        bless {
                doc => $doc,
                subject => $subj,
@@ -47,6 +47,7 @@ sub load_doc {
                references => $refs,
                to => $to,
                cc => $cc,
+               blob => $blob,
        }, $class;
 }
 
@@ -105,9 +106,11 @@ sub ts {
 }
 
 sub to_doc_data {
-       my ($self) = @_;
-       join("\n", $self->subject, $self->from, $self->references,
-               $self->to, $self->cc);
+       my ($self, $blob) = @_;
+       my @rows = ($self->subject, $self->from, $self->references,
+                       $self->to, $self->cc);
+       push @rows, $blob if defined $blob;
+       join("\n", @rows);
 }
 
 sub references {
@@ -185,6 +188,15 @@ sub mid ($;$) {
 
 sub _extract_mid { mid_clean(mid_mime($_[0]->mime)) }
 
+sub blob {
+       my ($self, $x40) = @_;
+       if (defined $x40) {
+               $self->{blob} = $x40;
+       } else {
+               $self->{blob};
+       }
+}
+
 sub mime {
        my ($self, $mime) = @_;
        if (defined $mime) {