search: avoid creating ghosts for circular References

Some mail software incorrectly creates circular references
and causes us to create ghosts before the actual mail doc
is created.

diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm
index 617c267b789534ad10a31d5da4c747197fbc41bf..db86301d8794e8bee3e83930724448983bc6247e 100644 (file)
--- a/lib/PublicInbox/Search.pm
+++ b/lib/PublicInbox/Search.pm
@@ -79,8 +79,8 @@ sub add_message {
        my $db = $self->{xdb};
 
        my $doc_id;
-       my $mid = mid_clean($mime->header_obj->header_raw('Message-ID'));
-       $mid = mid_compressed($mid);
+       my $mid_orig = mid_clean($mime->header_obj->header_raw('Message-ID'));
+       my $mid = mid_compressed($mid_orig);
        my $was_ghost = 0;
        my $ct_msg = $mime->header('Content-Type') || 'text/plain';
        my $enc_msg = PublicInbox::View::enc_for($ct_msg);
@@ -176,7 +176,7 @@ sub add_message {
        };
 
        if ($@) {
-               warn "failed to index message <$mid>: $@\n";
+               warn "failed to index message <$mid_orig>: $@\n";
                return undef;
        }
        $doc_id;
@@ -184,11 +184,11 @@ sub add_message {
 
 # returns deleted doc_id on success, undef on missing
 sub remove_message {
-       my ($self, $mid) = @_;
+       my ($self, $mid_orig) = @_;
        my $db = $self->{xdb};
        my $doc_id;
-       $mid = mid_clean($mid);
-       $mid = mid_compressed($mid);
+       $mid_orig = mid_clean($mid_orig);
+       my $mid = mid_compressed($mid_orig);
 
        eval {
                $doc_id = $self->find_unique_doc_id('mid', $mid);
@@ -196,7 +196,7 @@ sub remove_message {
        };
 
        if ($@) {
-               warn "failed to remove message <$mid>: $@\n";
+               warn "failed to remove message <$mid_orig>: $@\n";
                return undef;
        }
        $doc_id;
@@ -347,16 +347,33 @@ sub link_message_to_parents {
                if ($irt =~ /<([^>]+)>/) {
                        $irt = $1;
                }
-               push @refs, $irt;
+
+               # maybe some crazies will try to make a circular reference:
+               if ($irt eq $mid) {
+                       $irt = undef;
+               } else {
+                       push @refs, $irt;
+               }
        }
 
        my $tid;
        if (@refs) {
-               @refs = map { mid_compressed($_) } @refs;
-               my %uniq;
-               @refs = grep { !$uniq{$_}++ } @refs; # uniq
-
-               $doc->add_term(xpfx('inreplyto') . $refs[-1]);
+               my @crefs = map { mid_compressed($_) } @refs;
+               my %uniq = ($mid => 1);
+
+               # prevent circular references via References: here:
+               @refs = ();
+               foreach my $ref (@crefs) {
+                       next if $uniq{$ref};
+                       $uniq{$ref} = 1;
+                       push @refs, $ref;
+               }
+               $irt = undef if (defined $irt && !$uniq{$irt});
+       }
+       if (@refs) {
+               if (defined $irt) {
+                       $doc->add_term(xpfx('inreplyto') . $irt);
+               }
 
                my $ref_pfx = xpfx('references');
 

diff --git a/t/search.t b/t/search.t
index 0ad0886b7b859c2d018d7063374d6bbf65dda81c..55abe9e8bdd083d10105168731ae96f56aabcc55 100644 (file)
--- a/t/search.t
+++ b/t/search.t
@@ -243,6 +243,25 @@ sub filter_mids {
                "quoted result returned if nothing else");
 }
 
+# circular references
+{
+       my $doc_id = $rw->add_message(Email::MIME->create(
+               header_str => [
+                       Date => 'Sat, 02 Oct 2010 00:00:01 +0000',
+                       Subject => 'Circle',
+                       'Message-ID' => '<circle@a>',
+                       'References' => '<circle@a>',
+                       'In-Reply-To' => '<circle@a>',
+                       From => 'Circle <circle@example.com>',
+                       To => 'list@example.com',
+               ],
+               body => "LOOP!\n"));
+       ok($doc_id > 0, "doc_id defined with circular reference");
+       my $smsg = $rw->lookup_message('circle@a');
+       $smsg->ensure_metadata;
+       is($smsg->{references}, undef, "no references created");
+}
+
 done_testing();
 
 1;