X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FSearchThread.pm;h=8fb3a030aa723417036108bb1728d8abeec8be4f;hb=e3b57fe9f68e80fc85cff46ccec9246b670f1312;hp=53fec974f3ef51afb5466b9a715ef99a8bf62cd6;hpb=76081da737b238c9ba9d4683571f04865e598c64;p=public-inbox.git diff --git a/lib/PublicInbox/SearchThread.pm b/lib/PublicInbox/SearchThread.pm index 53fec974..8fb3a030 100644 --- a/lib/PublicInbox/SearchThread.pm +++ b/lib/PublicInbox/SearchThread.pm @@ -4,10 +4,10 @@ # This license differs from the rest of public-inbox # # Our own jwz-style threading class based on Mail::Thread from CPAN. -# Mail::Thread is unmaintained and available on some distros. +# Mail::Thread is unmaintained and unavailable on some distros. # We also do not want pruning or subject grouping, since we want # to encourage strict threading and hopefully encourage people -# to use proper In-Reply-To. +# to use proper In-Reply-To/References. # # This includes fixes from several open bugs for Mail::Thread # @@ -20,301 +20,161 @@ package PublicInbox::SearchThread; use strict; use warnings; -use Email::Abstract; - -sub new { - return bless { - messages => $_[1], - id_table => {}, - rootset => [] - }, $_[0]; -} - -sub _get_hdr { - my ($class, $msg, $hdr) = @_; - Email::Abstract->get_header($msg, $hdr) || ''; -} - -sub _uniq { - my %seen; - return grep { !$seen{$_}++ } @_; -} - -sub _references { - my $class = shift; - my $msg = shift; - my @references = ($class->_get_hdr($msg, "References") =~ /<([^>]+)>/g); - my $foo = $class->_get_hdr($msg, "In-Reply-To"); - chomp $foo; - $foo =~ s/.*?<([^>]+)>.*/$1/; - push @references, $foo - if $foo =~ /^\S+\@\S+$/ && (!@references || $references[-1] ne $foo); - return _uniq(@references); -} - -sub _msgid { - my ($class, $msg) = @_; - my $id = $class->_get_hdr($msg, "Message-ID"); - die "attempt to thread message with no id" unless $id; - chomp $id; - $id =~ s/^<([^>]+)>.*/$1/; # We expect this not to have <>s - return $id; -} - -sub rootset { @{$_[0]{rootset}} } +use PublicInbox::MID qw($MID_EXTRACT); sub thread { - my $self = shift; - $self->_setup(); - $self->{rootset} = [ - grep { !$_->{parent} } values %{$self->{id_table}} ]; - $self->_finish(); -} - -sub _finish { - my $self = shift; - delete $self->{id_table}; - delete $self->{seen}; -} - -sub _get_cont_for_id { - my $self = shift; - my $id = shift; - $self->{id_table}{$id} ||= $self->_container_class->new($id); -} - -sub _container_class { 'PublicInbox::SearchThread::Container' } - -sub _setup { - my ($self) = @_; - - _add_message($self, $_) foreach @{$self->{messages}}; -} - -sub _add_message ($$) { - my ($self, $message) = @_; - - # A. if id_table... - my $this_container = $self->_get_cont_for_id($self->_msgid($message)); - $this_container->{message} = $message; + my ($msgs, $ordersub, $ctx) = @_; + + # A. put all current $msgs (non-ghosts) into %id_table + my %id_table = map {; + # this delete saves around 4K across 1K messages + # TODO: move this to a more appropriate place, breaks tests + # if we do it during psgi_cull + delete $_->{num}; + + $_->{mid} => PublicInbox::SearchThread::Msg::cast($_); + } @$msgs; + + # Sadly, we sort here anyways since the fill-in-the-blanks References: + # can be shakier if somebody used In-Reply-To with multiple, disparate + # messages. So, take the client Date: into account since we can't + # always determine ordering when somebody uses multiple In-Reply-To. + # We'll trust the client Date: header here instead of the Received: + # time since this is for display (and not retrieval) + _set_parent(\%id_table, $_) for sort { $a->{ds} <=> $b->{ds} } @$msgs; + my $ibx = $ctx->{ibx}; + my $rootset = [ grep { + !delete($_->{parent}) && $_->visible($ibx) + } values %id_table ]; + $rootset = $ordersub->($rootset); + $_->order_children($ordersub, $ctx) for @$rootset; + $rootset; +} + +sub _set_parent ($$) { + my ($id_table, $this) = @_; # B. For each element in the message's References field: - my @refs = $self->_references($message); + defined(my $refs = $this->{references}) or return; + # This loop exists to help fill in gaps left from missing + # messages. It is not needed in a perfect world where + # everything is perfectly referenced, only the last ref + # matters. my $prev; - for my $ref (@refs) { + foreach my $ref ($refs =~ m/$MID_EXTRACT/go) { # Find a Container object for the given Message-ID - my $container = $self->_get_cont_for_id($ref); - - # Link the References field's Containers together in the - # order implied by the References header - # * If they are already linked don't change the existing links - # * Do not add a link if adding that link would introduce - # a loop... - + my $cont = $id_table->{$ref} //= + PublicInbox::SearchThread::Msg::ghost($ref); + + # Link the References field's Containers together in + # the order implied by the References header + # + # * If they are already linked don't change the + # existing links + # * Do not add a link if adding that link would + # introduce a loop... if ($prev && - !$container->{parent} && # already linked - !$container->has_descendent($prev) # would loop + !$cont->{parent} && # already linked + !$cont->has_descendent($prev) # would loop ) { - $prev->add_child($container); + $prev->add_child($cont); } - $prev = $container; + $prev = $cont; } # C. Set the parent of this message to be the last element in - # References... - if ($prev && - !$this_container->has_descendent($prev) # would loop - ) { - $prev->add_child($this_container) + # References. + if (defined $prev && !$this->has_descendent($prev)) { # would loop + $prev->add_child($this); } } -sub order { - my $self = shift; - my $ordersub = shift; - - # make a fake root - my $root = $self->_container_class->new( 'fakeroot' ); - $root->add_child( $_ ) for @{ $self->{rootset} }; - - # sort it - $root->order_children( $ordersub ); +package PublicInbox::SearchThread::Msg; +use base qw(PublicInbox::Smsg); +use strict; +use warnings; +use Carp qw(croak); - # and untangle it - my $kids = $root->children; - $self->{rootset} = $kids; - $root->remove_child($_) for @$kids; +# declare a ghost smsg (determined by absence of {blob}) +sub ghost { + bless { + mid => $_[0], + children => {}, # becomes an array when sorted by ->order(...) + }, __PACKAGE__; } -package PublicInbox::SearchThread::Container; -use Carp qw(croak); -use Scalar::Util qw(weaken); - -sub new { my $self = shift; bless { id => shift }, $self; } +# give a existing smsg the methods of this class +sub cast { + my ($smsg) = @_; + $smsg->{children} = {}; + bless $smsg, __PACKAGE__; +} -sub message { $_[0]->{message} } -sub child { $_[0]->{child} } -sub next { $_[0]->{next} } -sub messageid { $_[0]->{id} } +sub topmost { + my ($self) = @_; + my @q = ($self); + while (my $cont = shift @q) { + return $cont if $cont->{blob}; + push @q, values %{$cont->{children}}; + } + undef; +} sub add_child { my ($self, $child) = @_; croak "Cowardly refusing to become my own parent: $self" if $self == $child; - if (grep { $_ == $child } @{$self->children}) { - # All is potentially correct with the world - weaken($child->{parent} = $self); - return; - } - - my $parent = $child->{parent}; - remove_child($parent, $child) if $parent; - - $child->{next} = $self->{child}; - $self->{child} = $child; - weaken($child->{parent} = $self); -} - -sub remove_child { - my ($self, $child) = @_; + my $cid = $child->{mid}; - my $x = $self->{child} or return; - if ($x == $child) { # First one's easy. - $self->{child} = $child->{next}; - $child->{parent} = $child->{next} = undef; - return; + # reparenting: + if (defined(my $parent = $child->{parent})) { + delete $parent->{children}->{$cid}; } - my $prev = $x; - while ($x = $x->{next}) { - if ($x == $child) { - $prev->{next} = $x->{next}; # Unlink x - $x->{next} = $x->{parent} = undef; # Deparent it - return; - } - $prev = $x; - } - # oddly, we can get here - $child->{next} = $child->{parent} = undef; + $self->{children}->{$cid} = $child; + $child->{parent} = $self; } sub has_descendent { - my $self = shift; - my $child = shift; - die "Assertion failed: $child" unless eval {$child}; - my $there = 0; - $self->recurse_down(sub { $there = 1 if $_[0] == $child }); - - return $there; -} - -sub children { - my $self = shift; - my @children; - my $visitor = $self->{child}; - while ($visitor) { - push @children, $visitor; - $visitor = $visitor->{next}; + my ($self, $child) = @_; + my %seen; # loop prevention + while ($child) { + return 1 if $self == $child || $seen{$child}++; + $child = $child->{parent}; } - \@children; -} - -sub set_children { - my ($self, $children) = @_; - my $walk = $self->{child} = shift @$children; - do { - $walk = $walk->{next} = shift @$children; - } while ($walk); -} - -sub order_children { - my $self = shift; - my $ordersub = shift; - - return unless $ordersub; - - my $sub = sub { - my $cont = shift; - my $children = $cont->children; - return if @$children < 2; - $cont->set_children( $ordersub->( $children ) ); - }; - $self->iterate_down( undef, $sub ); - undef $sub; -} - -# non-recursive version of recurse_down to avoid stack depth warnings -sub recurse_down { - my ($self, $callback) = @_; - my %seen; - my @q = ($self); - while (my $cont = shift @q) { - $seen{$cont}++; - $callback->($cont); - - if (my $next = $cont->{next}) { - if ($seen{$next}) { - $cont->{next} = undef; - } else { - push @q, $next; - } - } - if (my $child = $cont->{child}) { - if ($seen{$child}) { - $cont->{child} = undef; - } else { - push @q, $child; - } - } + 0; +} + +# Do not show/keep ghosts iff they have no children. Sometimes +# a ghost Message-ID is the result of a long header line +# being folded/mangled by a MUA, and not a missing message. +sub visible ($$) { + my ($self, $ibx) = @_; + return 1 if $self->{blob}; + if (my $by_mid = $ibx->smsg_by_mid($self->{mid})) { + %$self = (%$self, %$by_mid); + 1; + } else { + (scalar values %{$self->{children}}); } } -sub iterate_down { - my $self = shift; - my ($before, $after) = @_; - - my %seen; - my $walk = $self; - my $depth = 0; - my @visited; - while ($walk) { - push @visited, [ $walk, $depth ]; - $before->($walk, $depth) if $before; - - # spot/break loops - $seen{$walk}++; - - my $child = $walk->{child}; - if ($child && $seen{$child}) { - $walk->{child} = $child = undef; - } - - my $next = $walk->{next}; - if ($next && $seen{$next}) { - $walk->{next} = $next = undef; - } - - # go down, or across - if ($child) { - $next = $child; - ++$depth; - } - - # no next? look up - if (!$next) { - my $up = $walk; - while ($up && !$next) { - $up = $up->{parent}; - --$depth; - $next = $up->{next} if $up; - } - } - $walk = $next; +sub order_children { + my ($cur, $ordersub, $ctx) = @_; + + my %seen = ($cur => 1); # self-referential loop prevention + my @q = ($cur); + my $ibx = $ctx->{ibx}; + while (defined($cur = shift @q)) { + my $c = $cur->{children}; # The hashref here... + + $c = [ grep { !$seen{$_}++ && visible($_, $ibx) } values %$c ]; + $c = $ordersub->($c) if scalar @$c > 1; + $cur->{children} = $c; # ...becomes an arrayref + push @q, @$c; } - return unless $after; - while (@visited) { $after->(@{ pop @visited }) } } 1;