From: Eric Wong Date: Mon, 17 Aug 2015 03:11:43 +0000 (+0000) Subject: Merge remote-tracking branch 'origin/search' X-Git-Tag: v1.0.0~1066 X-Git-Url: http://www.git.stargrave.org/?p=public-inbox.git;a=commitdiff_plain;h=118ac5c2dc0e7a193ff58f2a9003636e1353583c;hp=af0c513d49bae691f636b00efd1635b565b40c9d Merge remote-tracking branch 'origin/search' * origin/search: view: deduplicate common code for loading search results SearchMsg: ensure metadata for ghost messages mid implement /s/$SUBJECT_PATH.html lookups search: remove unnecessary xpfx export www: /t/$MESSAGE_ID.html for threads view: hoist out index_walk function view: reply threading adjustment thread: common sorting code view: display replies in per-message view search: make search results more OO extract redundant Message-ID handling code search: implement index_sync to fixup indexer initial search backend implementation --- diff --git a/Makefile.PL b/Makefile.PL index 1ee10896..f302b7c0 100644 --- a/Makefile.PL +++ b/Makefile.PL @@ -9,7 +9,8 @@ WriteMakefile( AUTHOR => 'Eric Wong ', ABSTRACT => 'public-inbox server infrastructure', EXE_FILES => [qw/public-inbox-mda public-inbox.cgi - public-inbox-learn public-inbox-init/], + public-inbox-learn public-inbox-init + public-inbox-index/], PREREQ_PM => { # note: we use ssoma(1) and spamc(1), NOT the Perl modules # We also depend on git through ssoma. diff --git a/lib/PublicInbox/Feed.pm b/lib/PublicInbox/Feed.pm index f7c2f329..b5325597 100644 --- a/lib/PublicInbox/Feed.pm +++ b/lib/PublicInbox/Feed.pm @@ -73,11 +73,13 @@ sub generate_html_index { '' . PRE_WRAP; # sort child messages in chronological order - $th->order(sub { mime_sort_children(@_) }); + $th->order(*PublicInbox::Thread::sort_ts); # except we sort top-level messages reverse chronologically my $state = [ time, {}, $first, 0 ]; - for (mime_sort_roots($th)) { dump_msg($_, 0, \$html, $state) } + for (PublicInbox::Thread::rsort_ts($th->rootset)) { + dump_msg($_, 0, \$html, $state) + } Email::Address->purge_cache; my $footer = nav_footer($args->{cgi}, $last, $feed_opts, $state); @@ -299,26 +301,9 @@ sub mime_load_for_sort { my $t = eval { str2time($mime->header('Date')) }; defined($t) or $t = 0; - $mime->header_set('X-PI-Date', $t); + $mime->header_set('X-PI-TS', $t); push @$messages, $mime; 1; } -# children are chronological -sub mime_sort_children { - sort { - $a->topmost->message->header('X-PI-Date') <=> - $b->topmost->message->header('X-PI-Date') - } @_; -} - -# parents are reverse chronological -sub mime_sort_roots { - my ($th) = @_; - sort { - (eval { $b->message->header('X-PI-Date') } || 0) <=> - (eval { $a->message->header('X-PI-Date') } || 0) - } $th->rootset; -} - 1; diff --git a/lib/PublicInbox/Hval.pm b/lib/PublicInbox/Hval.pm index 68f89546..d8b31c84 100644 --- a/lib/PublicInbox/Hval.pm +++ b/lib/PublicInbox/Hval.pm @@ -8,6 +8,7 @@ use warnings; use fields qw(raw href); use Encode qw(find_encoding); use URI::Escape qw(uri_escape_utf8); +use PublicInbox::MID qw/mid_clean mid_compressed/; my $enc_ascii = find_encoding('us-ascii'); @@ -25,16 +26,8 @@ sub new { sub new_msgid { my ($class, $msgid) = @_; - $msgid =~ s/\A\s*?\s*\z//; - - if (length($msgid) <= 40) { - $class->new($msgid); - } else { - require Digest::SHA; - my $hex = Digest::SHA::sha1_hex($msgid); - $class->new($msgid, $hex); - } + $msgid = mid_clean($msgid); + $class->new($msgid, mid_compressed($msgid)); } sub new_oneline { diff --git a/lib/PublicInbox/MID.pm b/lib/PublicInbox/MID.pm new file mode 100644 index 00000000..d097011b --- /dev/null +++ b/lib/PublicInbox/MID.pm @@ -0,0 +1,39 @@ +# Copyright (C) 2015, all contributors +# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt) +package PublicInbox::MID; +use strict; +use warnings; +use base qw/Exporter/; +our @EXPORT_OK = qw/mid_clean mid_compressed mid2path/; +use Digest::SHA qw/sha1_hex/; +use constant MID_MAX => 40; # SHA-1 hex length + +sub mid_clean { + my ($mid) = @_; + defined($mid) or die "no Message-ID"; + # MDA->precheck did more checking for us + $mid =~ s/\A\s*?\s*\z//; + $mid; +} + +# this is idempotent +sub mid_compressed { + my ($mid) = @_; + return $mid if (length($mid) <= MID_MAX); + sha1_hex($mid); +} + +sub mid2path { + my ($mid) = @_; + my ($x2, $x38) = ($mid =~ /\A([a-f0-9]{2})([a-f0-9]{38})\z/); + + unless (defined $x38) { + # compatibility with old links (or short Message-IDs :) + $mid = sha1_hex($mid); + ($x2, $x38) = ($mid =~ /\A([a-f0-9]{2})([a-f0-9]{38})\z/); + } + "$x2/$x38"; +} + +1; diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm new file mode 100644 index 00000000..f4f00b25 --- /dev/null +++ b/lib/PublicInbox/Search.pm @@ -0,0 +1,549 @@ +# Copyright (C) 2015, all contributors +# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt) +# based on notmuch, but with no concept of folders, files or flags +package PublicInbox::Search; +use strict; +use warnings; +use PublicInbox::SearchMsg; +use Search::Xapian qw/:standard/; +require PublicInbox::View; +use Email::MIME; +use PublicInbox::MID qw/mid_clean mid_compressed/; + +use constant { + TS => 0, + # SCHEMA_VERSION history + # 0 - initial + # 1 - subject_path is lower-cased + SCHEMA_VERSION => 1, + LANG => 'english', + QP_FLAGS => FLAG_PHRASE|FLAG_BOOLEAN|FLAG_LOVEHATE|FLAG_WILDCARD, +}; + +# setup prefixes +my %bool_pfx_internal = ( + type => 'T', # "mail" or "ghost" + mid => 'Q', # uniQue id (Message-ID or mid_compressed) +); + +my %bool_pfx_external = ( + path => 'XPATH', + thread => 'G', # newsGroup (or similar entity - e.g. a web forum name) + references => 'XREFS', + inreplyto => 'XIRT', +); + +my %prob_prefix = ( + subject => 'S', +); + +my %all_pfx = (%bool_pfx_internal, %bool_pfx_external, %prob_prefix); + +sub xpfx { $all_pfx{$_[0]} } + +our %PFX2TERM_RMAP; +while (my ($k, $v) = each %all_pfx) { + next if $prob_prefix{$k}; + $PFX2TERM_RMAP{$v} = $k; +} + +my $mail_query = Search::Xapian::Query->new(xpfx('type') . 'mail'); + +sub new { + my ($class, $git_dir, $writable) = @_; + # allow concurrent versions for easier rollback: + my $dir = "$git_dir/public-inbox/xapian" . SCHEMA_VERSION; + my $db; + + if ($writable) { # not used by the WWW interface + require Search::Xapian::WritableDatabase; + require File::Path; + File::Path::mkpath($dir); + $db = Search::Xapian::WritableDatabase->new($dir, + Search::Xapian::DB_CREATE_OR_OPEN); + } else { + $db = Search::Xapian::Database->new($dir); + } + bless { xdb => $db, git_dir => $git_dir }, $class; +} + +sub reopen { $_[0]->{xdb}->reopen } + +sub add_message { + my ($self, $mime) = @_; # mime = Email::MIME object + my $db = $self->{xdb}; + + my $doc_id; + my $mid = mid_clean($mime->header('Message-ID')); + $mid = mid_compressed($mid); + my $was_ghost = 0; + my $ct_msg = $mime->header('Content-Type') || 'text/plain'; + my $enc_msg = PublicInbox::View::enc_for($ct_msg); + + $db->begin_transaction; + eval { + my $smsg = $self->lookup_message($mid); + my $doc; + + if ($smsg) { + $smsg->ensure_metadata; + # convert a ghost to a regular message + # it will also clobber any existing regular message + $smsg->mime($mime); + $doc = $smsg->{doc}; + + my $type = xpfx('type'); + eval { + $doc->remove_term($type . 'ghost'); + $was_ghost = 1; + }; + + # probably does not exist: + eval { $doc->remove_term($type . 'mail') }; + $doc->add_term($type . 'mail'); + } else { + $smsg = PublicInbox::SearchMsg->new($mime); + $doc = $smsg->{doc}; + $doc->add_term(xpfx('mid') . $mid); + } + + my $subj = $smsg->subject; + + if (length $subj) { + $doc->add_term(xpfx('subject') . $subj); + + my $path = subject_path($subj); + $doc->add_term(xpfx('path') . $path); + } + + my $from = $smsg->from_name; + my $date = $smsg->date; + my $ts = Search::Xapian::sortable_serialise($smsg->ts); + $doc->add_value(PublicInbox::Search::TS, $ts); + + $doc->set_data($smsg->to_doc_data); + + my $tg = $self->term_generator; + + $tg->set_document($doc); + $tg->index_text($subj, 1, 'S') if $subj; + $tg->increase_termpos; + $tg->index_text($subj) if $subj; + $tg->increase_termpos; + + $tg->index_text($smsg->from->format); + $tg->increase_termpos; + + $mime->walk_parts(sub { + my ($part) = @_; + return if $part->subparts; # walk_parts already recurses + my $ct = $part->content_type || $ct_msg; + + # account for filter bugs... + $ct =~ m!\btext/plain\b!i or return; + + my $enc = PublicInbox::View::enc_for($ct, $enc_msg); + my (@orig, @quot); + foreach my $l (split(/\n/, $enc->decode($part->body))) { + if ($l =~ /^\s*>/) { + push @quot, $l; + } else { + push @orig, $l; + } + } + if (@quot) { + $tg->index_text(join("\n", @quot), 0); + $tg->increase_termpos; + } + if (@orig) { + $tg->index_text(join("\n", @orig)); + $tg->increase_termpos; + } + }); + + if ($was_ghost) { + $doc_id = $smsg->doc_id; + $self->link_message($smsg, 0); + $db->replace_document($doc_id, $doc); + } else { + $self->link_message($smsg, 0); + $doc_id = $db->add_document($doc); + } + }; + + if ($@) { + warn "failed to index message <$mid>: $@\n"; + $db->cancel_transaction; + } else { + $db->commit_transaction; + } + $doc_id; +} + +# returns deleted doc_id on success, undef on missing +sub remove_message { + my ($self, $mid) = @_; + my $db = $self->{xdb}; + my $doc_id; + $mid = mid_clean($mid); + $mid = mid_compressed($mid); + + $db->begin_transaction; + eval { + $doc_id = $self->find_unique_doc_id('mid', $mid); + $db->delete_document($doc_id) if defined $doc_id; + }; + + if ($@) { + warn "failed to remove message <$mid>: $@\n"; + $db->cancel_transaction; + } else { + $db->commit_transaction; + } + $doc_id; +} + +# read-only +sub query { + my ($self, $query_string, $opts) = @_; + my $query = $self->qp->parse_query($query_string, QP_FLAGS); + + $query = Search::Xapian::Query->new(OP_AND, $mail_query, $query); + $self->do_enquire($query, $opts); +} + +sub get_subject_path { + my ($self, $path, $opts) = @_; + my $query = $self->qp->parse_query("path:$path", 0); + $self->do_enquire($query); +} + +# given a message ID, get replies to a message +sub get_replies { + my ($self, $mid, $opts) = @_; + $mid = mid_clean($mid); + $mid = mid_compressed($mid); + my $qp = $self->qp; + my $irt = $qp->parse_query("inreplyto:$mid", 0); + my $ref = $qp->parse_query("references:$mid", 0); + my $query = Search::Xapian::Query->new(OP_OR, $irt, $ref); + + $self->do_enquire($query); +} + +sub get_thread { + my ($self, $mid, $opts) = @_; + my $smsg = eval { $self->lookup_message($mid) }; + + return { count => 0, msgs => [] } unless $smsg; + my $qp = $self->qp; + my $qtid = $qp->parse_query('thread:'.$smsg->thread_id); + my $qsub = $qp->parse_query('path:'.$smsg->path); + my $query = Search::Xapian::Query->new(OP_OR, $qtid, $qsub); + $self->do_enquire($query); +} + +# private subs below + +sub do_enquire { + my ($self, $query, $opts) = @_; + my $enquire = $self->enquire; + + $enquire->set_query($query); + $enquire->set_sort_by_relevance_then_value(TS, 0); + $opts ||= {}; + my $offset = $opts->{offset} || 0; + my $limit = $opts->{limit} || 50; + my $mset = $enquire->get_mset($offset, $limit); + my @msgs = map { + PublicInbox::SearchMsg->load_doc($_->get_document); + } $mset->items; + + { count => $mset->get_matches_estimated, msgs => \@msgs } +} + +# read-write +sub stemmer { Search::Xapian::Stem->new(LANG) } + +# read-only +sub qp { + my ($self) = @_; + + my $qp = $self->{query_parser}; + return $qp if $qp; + + # new parser + $qp = Search::Xapian::QueryParser->new; + $qp->set_default_op(OP_AND); + $qp->set_database($self->{xdb}); + $qp->set_stemmer($self->stemmer); + $qp->set_stemming_strategy(STEM_SOME); + $qp->add_valuerangeprocessor($self->ts_range_processor); + $qp->add_valuerangeprocessor($self->date_range_processor); + + while (my ($name, $prefix) = each %bool_pfx_external) { + $qp->add_boolean_prefix($name, $prefix); + } + + while (my ($name, $prefix) = each %prob_prefix) { + $qp->add_prefix($name, $prefix); + } + + $self->{query_parser} = $qp; +} + +sub term_generator { # write-only + my ($self) = @_; + + my $tg = $self->{term_generator}; + return $tg if $tg; + + $tg = Search::Xapian::TermGenerator->new; + $tg->set_stemmer($self->stemmer); + + $self->{term_generator} = $tg; +} + +sub next_doc_id { $_[0]->{xdb}->get_lastdocid + 1 } + +# increments last_thread_id counter +# returns a 64-bit integer represented as a hex string +sub next_thread_id { + my ($self) = @_; + my $db = $self->{xdb}; + my $last_thread_id = int($db->get_metadata('last_thread_id') || 0); + + $db->set_metadata('last_thread_id', ++$last_thread_id); + + $last_thread_id; +} + +sub ts_range_processor { + $_[0]->{tsrp} ||= Search::Xapian::NumberValueRangeProcessor->new(TS); +} + +sub date_range_processor { + $_[0]->{drp} ||= Search::Xapian::DateValueRangeProcessor->new(TS); +} + +sub link_message { + my ($self, $smsg, $is_ghost) = @_; + + if ($is_ghost) { + $smsg->ensure_metadata; + } else { + $self->link_message_to_parents($smsg); + } +} + +sub link_message_to_parents { + my ($self, $smsg) = @_; + my $doc = $smsg->{doc}; + my $mid = mid_compressed($smsg->mid); + my $mime = $smsg->mime; + my $refs = $mime->header('References'); + my @refs = $refs ? ($refs =~ /<([^>]+)>/g) : (); + my $irt = $mime->header('In-Reply-To'); + if ($irt) { + if ($irt =~ /<([^>]+)>/) { + $irt = $1; + } + push @refs, $irt; + } + + my $tid; + if (@refs) { + @refs = map { mid_compressed($_) } @refs; + my %uniq; + @refs = grep { !$uniq{$_}++ } @refs; # uniq + + $doc->add_term(xpfx('inreplyto') . $refs[-1]); + + my $ref_pfx = xpfx('references'); + + # first ref *should* be the thread root, + # but we can never trust clients to do the right thing + my $ref = shift @refs; + $doc->add_term($ref_pfx . $ref); + $tid = $self->_resolve_mid_to_tid($ref); + + # the rest of the refs should point to this tid: + foreach $ref (@refs) { + $doc->add_term($ref_pfx . $ref); + my $ptid = $self->_resolve_mid_to_tid($ref); + if ($tid ne $ptid) { + $self->merge_threads($tid, $ptid); + } + } + } else { + $tid = $self->next_thread_id; + } + $doc->add_term(xpfx('thread') . $tid); +} + +sub lookup_message { + my ($self, $mid) = @_; + $mid = mid_clean($mid); + $mid = mid_compressed($mid); + + my $doc_id = $self->find_unique_doc_id('mid', $mid); + my $smsg; + if (defined $doc_id) { + # raises on error: + my $doc = $self->{xdb}->get_document($doc_id); + $smsg = PublicInbox::SearchMsg->wrap($doc, $mid); + $smsg->doc_id($doc_id); + } + $smsg; +} + +sub find_unique_doc_id { + my ($self, $term, $value) = @_; + + my ($begin, $end) = $self->find_doc_ids($term, $value); + + return undef if $begin->equal($end); # not found + + my $rv = $begin->get_docid; + + # sanity check + $begin->inc; + $begin->equal($end) or die "Term '$term:$value' is not unique\n"; + $rv; +} + +# returns begin and end PostingIterator +sub find_doc_ids { + my ($self, $term, $value) = @_; + + $self->find_doc_ids_for_term(xpfx($term) . $value); +} + +# returns begin and end PostingIterator +sub find_doc_ids_for_term { + my ($self, $term) = @_; + my $db = $self->{xdb}; + + ($db->postlist_begin($term), $db->postlist_end($term)); +} + +# this will create a ghost as necessary +sub _resolve_mid_to_tid { + my ($self, $mid) = @_; + + my $smsg = $self->lookup_message($mid) || $self->create_ghost($mid); + $smsg->thread_id; +} + +sub create_ghost { + my ($self, $mid, $tid) = @_; + + $mid = mid_compressed($mid); + $tid = $self->next_thread_id unless defined $tid; + + my $doc = Search::Xapian::Document->new; + $doc->add_term(xpfx('mid') . $mid); + $doc->add_term(xpfx('thread') . $tid); + $doc->add_term(xpfx('type') . 'ghost'); + + my $smsg = PublicInbox::SearchMsg->wrap($doc, $mid); + $self->link_message($smsg, 1); + $self->{xdb}->add_document($doc); + + $smsg; +} + +sub merge_threads { + my ($self, $winner_tid, $loser_tid) = @_; + my ($head, $tail) = $self->find_doc_ids('thread', $loser_tid); + my $thread_pfx = xpfx('thread'); + my $db = $self->{xdb}; + + for (; $head != $tail; $head->inc) { + my $docid = $head->get_docid; + my $doc = $db->get_document($docid); + $doc->remove_term($thread_pfx . $loser_tid); + $doc->add_term($thread_pfx . $winner_tid); + $db->replace_document($docid, $doc); + } +} + +# normalize subjects so they are suitable as pathnames for URLs +sub subject_path { + my $subj = pop; + + $subj =~ s/\A\s+//; + $subj =~ s/\s+\z//; + $subj =~ s/^(?:re|aw):\s*//i; # remove reply prefix (aw: German) + $subj =~ s![^a-zA-Z0-9_\.~/\-]+!_!g; + lc($subj); +} + +sub do_cat_mail { + my ($git, $blob) = @_; + my $mime = eval { + my $str = $git->cat_file($blob); + Email::MIME->new($str); + }; + $@ ? undef : $mime; +} + +sub index_blob { + my ($self, $git, $blob) = @_; + my $mime = do_cat_mail($git, $blob) or return; + eval { $self->add_message($mime) }; + warn "W: index_blob $blob: $@\n" if $@; +} + +sub unindex_blob { + my ($self, $git, $blob) = @_; + my $mime = do_cat_mail($git, $blob) or return; + my $mid = $mime->header('Message-ID'); + eval { $self->remove_message($mid) } if defined $mid; + warn "W: unindex_blob $blob: $@\n" if $@; +} + +sub enquire { + my ($self) = @_; + $self->{enquire} ||= Search::Xapian::Enquire->new($self->{xdb}); +} + +# indexes all unindexed messages +sub index_sync { + my ($self, $git) = @_; + my $db = $self->{xdb}; + my $latest = $db->get_metadata('last_commit'); + my $range = length $latest ? "$latest..HEAD" : 'HEAD'; + $latest = undef; + + my $hex = '[a-f0-9]'; + my $h40 = $hex .'{40}'; + my $addmsg = qr!^:000000 100644 \S+ ($h40) A\t${hex}{2}/${hex}{38}$!; + my $delmsg = qr!^:100644 000000 ($h40) \S+ D\t${hex}{2}/${hex}{38}$!; + + # get indexed messages + my @cmd = ('git', "--git-dir=$git->{git_dir}", "log", + qw/--reverse --no-notes --no-color --raw -r --no-abbrev/, + $range); + + my $pid = open(my $log, '-|', @cmd) or + die('open` '.join(' ', @cmd) . " pipe failed: $!\n"); + my $last; + while (my $line = <$log>) { + if ($line =~ /$addmsg/o) { + $self->index_blob($git, $1); + } elsif ($line =~ /$delmsg/o) { + $self->unindex_blob($git, $1); + } elsif ($line =~ /^commit ($h40)/o) { + my $commit = $1; + if (defined $latest) { + $db->set_metadata('last_commit', $latest) + } + $latest = $commit; + } + } + close $log; + $db->set_metadata('last_commit', $latest) if defined $latest; +} + +1; diff --git a/lib/PublicInbox/SearchMsg.pm b/lib/PublicInbox/SearchMsg.pm new file mode 100644 index 00000000..14a62eb6 --- /dev/null +++ b/lib/PublicInbox/SearchMsg.pm @@ -0,0 +1,205 @@ +# Copyright (C) 2015, all contributors +# License: GPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt) +# based on notmuch, but with no concept of folders, files or flags +package PublicInbox::SearchMsg; +use strict; +use warnings; +use Search::Xapian; +use Email::Address qw//; +use Email::Simple qw//; +use POSIX qw//; +use Date::Parse qw/str2time/; +use PublicInbox::MID qw/mid_clean mid_compressed/; +use Encode qw/find_encoding/; +my $enc_utf8 = find_encoding('UTF-8'); +our $PFX2TERM_RE = undef; + +sub new { + my ($class, $mime) = @_; + my $doc = Search::Xapian::Document->new; + $doc->add_term(PublicInbox::Search::xpfx('type') . 'mail'); + + bless { type => 'mail', doc => $doc, mime => $mime }, $class; +} + +sub wrap { + my ($class, $doc, $mid) = @_; + bless { doc => $doc, mime => undef, mid => $mid }, $class; +} + +sub load_doc { + my ($class, $doc) = @_; + my $data = $doc->get_data; + $data = $enc_utf8->decode($data); + my ($mid, $subj, $from, $date) = split(/\n/, $data); + bless { + doc => $doc, + mid => $mid, + subject => $subj, + date => $date, + from_name => $from, + }, $class; +} + +sub subject { + my ($self) = @_; + my $subj = $self->{subject}; + return $subj if defined $subj; + $subj = $self->{mime}->header('Subject'); + $subj = '' unless defined $subj; + $subj =~ tr/\n/ /; + $self->{subject} = $subj; +} + +sub from { + my ($self) = @_; + my $from = $self->mime->header('From') || ''; + my @from; + + if ($from) { + $from =~ tr/\n/ /; + @from = Email::Address->parse($from); + $self->{from} = $from[0]; + $from = $from[0]->name; + } + $self->{from_name} = $from; + $self->{from}; +} + +sub from_name { + my ($self) = @_; + my $from_name = $self->{from_name}; + return $from_name if defined $from_name; + $self->from; + $self->{from_name}; +} + +sub ts { + my ($self) = @_; + my $ts = $self->{ts}; + return $ts if $ts; + $self->{date} = undef; + $self->date; + $self->{ts}; +} + +sub date { + my ($self) = @_; + my $date = $self->{date}; + return $date if $date; + my $ts = eval { str2time($self->mime->header('Date')) } || 0; + $self->{ts} = $ts; + $self->{date} = POSIX::strftime('%Y-%m-%d %H:%M', gmtime($ts)); +} + +sub to_doc_data { + my ($self) = @_; + + $self->mid . "\n" . + $self->subject . "\n" . + $self->from_name . "\n". + $self->date; +} + +sub ensure_metadata { + my ($self) = @_; + my $doc = $self->{doc}; + my $i = $doc->termlist_begin; + my $end = $doc->termlist_end; + + unless (defined $PFX2TERM_RE) { + my $or = join('|', keys %PublicInbox::Search::PFX2TERM_RMAP); + $PFX2TERM_RE = qr/\A($or)/; + } + + for (; $i != $end; $i->inc) { + my $val = $i->get_termname; + + if ($val =~ s/$PFX2TERM_RE//o) { + my $field = $PublicInbox::Search::PFX2TERM_RMAP{$1}; + if ($field eq 'references') { + my $refs = $self->{references} ||= []; + push @$refs, $val; + } else { + $self->{$field} = $val; + } + } + } +} + +# for threading only +sub mini_mime { + my ($self) = @_; + $self->ensure_metadata; + my @h = ( + Subject => $self->subject, + 'X-PI-From' => $self->from_name, + 'X-PI-Date' => $self->date, + 'X-PI-TS' => $self->ts, + 'Message-ID' => "<$self->{mid}>", + ); + if (my $refs = $self->{references}) { + push @h, References => '<' . join('> <', @$refs) . '>'; + } + if (my $irt = $self->{inreplyto}) { + push @h, 'In-Reply-To' => "<$irt>"; + } + + Email::MIME->create(header_str => \@h); +} + +sub mid { + my ($self, $mid) = @_; + + if (defined $mid) { + $self->{mid} = $mid; + } else { + $self->ensure_metadata; # needed for ghosts + $self->{mid} ||= $self->_extract_mid; + } +} + +sub _extract_mid { + my ($self) = @_; + + my $mid = $self->mime->header('Message-ID'); + $mid ? mid_compressed(mid_clean($mid)) : $mid; +} + +sub mime { + my ($self, $mime) = @_; + if (defined $mime) { + $self->{mime} = $mime; + } else { + # TODO load from git + $self->{mime}; + } +} + +sub doc_id { + my ($self, $doc_id) = @_; + if (defined $doc_id) { + $self->{doc_id} = $doc_id; + } else { + # TODO load from xapian + $self->{doc_id}; + } +} + +sub thread_id { + my ($self) = @_; + my $tid = $self->{thread}; + return $tid if defined $tid; + $self->ensure_metadata; + $self->{thread}; +} + +sub path { + my ($self) = @_; + my $path = $self->{path}; + return $path if defined $path; + $self->ensure_metadata; + $self->{path}; +} + +1; diff --git a/lib/PublicInbox/Thread.pm b/lib/PublicInbox/Thread.pm index 7dabf243..58efb8dc 100644 --- a/lib/PublicInbox/Thread.pm +++ b/lib/PublicInbox/Thread.pm @@ -12,6 +12,20 @@ if ($Mail::Thread::VERSION <= 2.55) { eval q(sub _container_class { 'PublicInbox::Thread::Container' }); } +sub sort_ts { + sort { + (eval { $a->topmost->message->header('X-PI-TS') } || 0) <=> + (eval { $b->topmost->message->header('X-PI-TS') } || 0) + } @_; +} + +sub rsort_ts { + sort { + (eval { $b->topmost->message->header('X-PI-TS') } || 0) <=> + (eval { $a->topmost->message->header('X-PI-TS') } || 0) + } @_; +} + package PublicInbox::Thread::Container; use strict; use warnings; diff --git a/lib/PublicInbox/View.pm b/lib/PublicInbox/View.pm index 6e4d1789..47082431 100644 --- a/lib/PublicInbox/View.pm +++ b/lib/PublicInbox/View.pm @@ -3,17 +3,21 @@ package PublicInbox::View; use strict; use warnings; -use PublicInbox::Hval; use URI::Escape qw/uri_escape_utf8/; +use Date::Parse qw/str2time/; use Encode qw/find_encoding/; use Encode::MIME::Header; use Email::MIME::ContentType qw/parse_content_type/; +use PublicInbox::Hval; +use PublicInbox::MID qw/mid_clean mid_compressed mid2path/; +use Digest::SHA; require POSIX; # TODO: make these constants tunable use constant MAX_INLINE_QUOTED => 12; # half an 80x24 terminal use constant MAX_TRUNC_LEN => 72; use constant PRE_WRAP => ""; +use constant T_ANCHOR => '#u'; *ascii_html = *PublicInbox::Hval::ascii_html; @@ -21,16 +25,16 @@ my $enc_utf8 = find_encoding('UTF-8'); # public functions: sub msg_html { - my ($class, $mime, $full_pfx, $footer) = @_; + my ($class, $mime, $full_pfx, $footer, $srch) = @_; if (defined $footer) { $footer = "\n" . $footer; } else { $footer = ''; } - headers_to_html_header($mime, $full_pfx) . + headers_to_html_header($mime, $full_pfx, $srch) . multipart_text_as_html($mime, $full_pfx) . '
' . PRE_WRAP . - html_footer($mime, 1) . $footer . + html_footer($mime, 1, $full_pfx, $srch) . $footer . ''; } @@ -41,9 +45,10 @@ sub feed_entry { } # this is already inside a 
+# state = [ time, seen = {}, first_commit, page_nr = 0 ]
 sub index_entry {
-	my ($class, $mime, $level, $state) = @_;
-	my ($now, $seen, $first) = @$state;
+	my (undef, $mime, $level, $state) = @_;
+	my (undef, $seen, $first_commit) = @$state;
 	my $midx = $state->[3]++;
 	my ($prev, $next) = ($midx - 1, $midx + 1);
 	my $rv = '';
@@ -65,8 +70,17 @@ sub index_entry {
 	$from = PublicInbox::Hval->new_oneline($from)->as_html;
 	$subj = PublicInbox::Hval->new_oneline($subj)->as_html;
 	my $pfx = ('  ' x $level);
+	my $root_anchor = $seen->{root_anchor};
+	my $path;
+	my $more = 'permalink';
+	if ($root_anchor) {
+		$path = '../';
+		$subj = "$subj" if $root_anchor eq $id;
+	} else {
+		$path = '';
+	}
 
-	my $ts = $mime->header('X-PI-Date');
+	my $ts = $mime->header('X-PI-TS');
 	my $fmt = '%Y-%m-%d %H:%M UTC';
 	$ts = POSIX::strftime($fmt, gmtime($ts));
 
@@ -78,16 +92,18 @@ sub index_entry {
 	}
 	$rv .= "\n\n";
 
-	my $irp = $header_obj->header_raw('In-Reply-To');
-	my ($anchor_idx, $anchor);
-	if (defined $irp) {
-		$anchor_idx = anchor_for($irp);
+	my $irt = $header_obj->header_raw('In-Reply-To');
+	my ($anchor_idx, $anchor, $t_anchor);
+	if (defined $irt) {
+		$anchor_idx = anchor_for($irt);
 		$anchor = $seen->{$anchor_idx};
+		$t_anchor = T_ANCHOR;
+	} else {
+		$t_anchor = '';
 	}
 	my $href = $mid->as_href;
-	my $mhref = "m/$href.html";
-	my $fhref = "f/$href.html";
-	my $more = 'message';
+	my $mhref = "${path}m/$href.html";
+	my $fhref = "${path}f/$href.html";
 	# scan through all parts, looking for displayable text
 	$mime->walk_parts(sub {
 		$rv .= index_walk($_[0], $pfx, $enc_msg, $part_nr, $fhref,
@@ -96,24 +112,67 @@ sub index_entry {
 	});
 
 	$rv .= "\n$pfx$more ";
-	my $txt = "m/$href.txt";
+	my $txt = "${path}m/$href.txt";
 	$rv .= "raw ";
 	$rv .= html_footer($mime, 0);
 
-	if (defined $irp) {
+	if (defined $irt) {
 		unless (defined $anchor) {
-			my $v = PublicInbox::Hval->new_msgid($irp);
-			my $html = $v->as_html;
-			$anchor = 'm/' . $v->as_href . '.html';
+			my $v = PublicInbox::Hval->new_msgid($irt);
+			$v = $v->as_href;
+			$anchor = "${path}m/$v.html";
 			$seen->{$anchor_idx} = $anchor;
 		}
 		$rv .= " parent";
 	}
-	$rv .= " threadlink";
+
+	if ($first_commit) {
+		$rv .= " thread";
+	}
 
 	$rv . "\n\n";
 }
 
+sub thread_html {
+	my (undef, $ctx, $foot, $srch) = @_;
+	my $mid = mid_compressed($ctx->{mid});
+	my $res = $srch->get_thread($mid);
+	my $rv = '';
+	my $msgs = load_results($ctx, $res);
+	my $nr = scalar @$msgs;
+	return $rv if $nr == 0;
+	require PublicInbox::Thread;
+	my $th = PublicInbox::Thread->new(@$msgs);
+	$th->thread;
+	$th->order(*PublicInbox::Thread::sort_ts);
+	my $state = [ undef, { root_anchor => anchor_for($mid) }, undef, 0 ];
+	thread_entry(\$rv, $state, $_, 0) for $th->rootset;
+	my $final_anchor = $state->[3];
+	my $next = "end of thread\n";
+
+	$rv .= "
" . PRE_WRAP . $next . $foot . ""; +} + +sub subject_path_html { + my (undef, $ctx, $foot, $srch) = @_; + my $path = $ctx->{subject_path}; + my $res = $srch->get_subject_path($path); + my $rv = ''; + my $msgs = load_results($ctx, $res); + my $nr = scalar @$msgs; + return $rv if $nr == 0; + require PublicInbox::Thread; + my $th = PublicInbox::Thread->new(@$msgs); + $th->thread; + $th->order(*PublicInbox::Thread::sort_ts); + my $state = [ undef, { root_anchor => 'dummy' }, undef, 0 ]; + thread_entry(\$rv, $state, $_, 0) for $th->rootset; + my $final_anchor = $state->[3]; + my $next = "end of thread\n"; + + $rv .= "
" . PRE_WRAP . $next . $foot . ""; +} + # only private functions below. sub index_walk { @@ -173,7 +232,7 @@ sub enc_for { } sub multipart_text_as_html { - my ($mime, $full_pfx) = @_; + my ($mime, $full_pfx, $srch) = @_; my $rv = ""; my $part_nr = 0; my $enc_msg = enc_for($mime->header("Content-Type")); @@ -277,7 +336,7 @@ sub add_text_body_full { } sub headers_to_html_header { - my ($mime, $full_pfx) = @_; + my ($mime, $full_pfx, $srch) = @_; my $rv = ""; my @title; @@ -285,33 +344,34 @@ sub headers_to_html_header { my $v = $mime->header($h); defined($v) && length($v) or next; $v = PublicInbox::Hval->new_oneline($v); - $rv .= "$h: " . $v->as_html . "\n"; if ($h eq 'From') { my @from = Email::Address->parse($v->raw); - $v = $from[0]->name; - unless (defined($v) && length($v)) { - $v = '<' . $from[0]->address . '>'; - } - $title[1] = ascii_html($v); + $title[1] = ascii_html($from[0]->name); } elsif ($h eq 'Subject') { $title[0] = $v->as_html; + if ($srch) { + my $path = $srch->subject_path($v->raw); + $rv .= "$h: "; + $rv .= $v->as_html . "\n"; + next; + } } + $rv .= "$h: " . $v->as_html . "\n"; + } my $header_obj = $mime->header_obj; my $mid = $header_obj->header_raw('Message-ID'); - if (defined $mid) { - $mid = PublicInbox::Hval->new_msgid($mid); - $rv .= 'Message-ID: <' . $mid->as_html . '> '; - my $href = $mid->as_href; - $href = "../m/$href" unless $full_pfx; - $rv .= "(raw)\n"; - } + $mid = PublicInbox::Hval->new_msgid($mid); + $rv .= 'Message-ID: <' . $mid->as_html . '> '; + my $href = $mid->as_href; + $href = "../m/$href" unless $full_pfx; + $rv .= "(raw)\n"; - my $irp = $header_obj->header_raw('In-Reply-To'); - if (defined $irp) { - my $v = PublicInbox::Hval->new_msgid($irp); + my $irt = $header_obj->header_raw('In-Reply-To'); + if (defined $irt) { + my $v = PublicInbox::Hval->new_msgid($irt); my $html = $v->as_html; my $href = $v->as_href; $rv .= "In-Reply-To: <"; @@ -320,7 +380,7 @@ sub headers_to_html_header { my $refs = $header_obj->header_raw('References'); if ($refs) { - $refs =~ s/\s*\Q$irp\E\s*// if (defined $irp); + $refs =~ s/\s*\Q$irt\E\s*// if (defined $irt); my @refs = ($refs =~ /<([^>]+)>/g); if (@refs) { $rv .= 'References: '. linkify_refs(@refs) . "\n"; @@ -334,7 +394,7 @@ sub headers_to_html_header { } sub html_footer { - my ($mime, $standalone) = @_; + my ($mime, $standalone, $full_pfx, $srch) = @_; my %cc; # everyone else my $to; # this is the From address @@ -353,18 +413,41 @@ sub html_footer { my $subj = $mime->header('Subject') || ''; $subj = "Re: $subj" unless $subj =~ /\bRe:/; - my $irp = uri_escape_utf8( - $mime->header_obj->header_raw('Message-ID') || ''); + my $mid = $mime->header_obj->header_raw('Message-ID'); + my $irt = uri_escape_utf8($mid); delete $cc{$to}; $to = uri_escape_utf8($to); $subj = uri_escape_utf8($subj); my $cc = uri_escape_utf8(join(',', sort values %cc)); - my $href = "mailto:$to?In-Reply-To=$irp&Cc=${cc}&Subject=$subj"; + my $href = "mailto:$to?In-Reply-To=$irt&Cc=${cc}&Subject=$subj"; my $idx = $standalone ? " index" : ''; + if ($idx && $srch) { + $irt = $mime->header_obj->header_raw('In-Reply-To') || ''; + $mid = mid_compressed(mid_clean($mid)); + my $t_anchor = length $irt ? T_ANCHOR : ''; + $idx = " thread$idx"; + my $res = $srch->get_replies($mid); + if (my $c = $res->{count}) { + $c = $c == 1 ? '1 reply' : "$c replies"; + $idx .= "\n$c:\n"; + thread_replies(\$idx, $mime, $res); + } else { + $idx .= "\n(no replies yet)\n"; + } + if ($irt) { + $irt = PublicInbox::Hval->new_msgid($irt); + $irt = $irt->as_href; + $irt = "parent "; + } else { + $irt = ' ' x length('parent '); + } + } else { + $irt = ''; + } - "reply' . $idx; + "$irtreply' . $idx; } sub linkify_refs { @@ -376,12 +459,118 @@ sub linkify_refs { } @_); } -require Digest::SHA; sub anchor_for { my ($msgid) = @_; - $msgid =~ s/\A\s*?\s*\z//; - 'm' . Digest::SHA::sha1_hex($msgid); + 'm' . mid_compressed(mid_clean($msgid)); +} + +sub simple_dump { + my ($dst, $root, $node, $level) = @_; + my $pfx = ' ' x $level; + $$dst .= $pfx; + if (my $x = $node->message) { + my $mid = $x->header('Message-ID'); + if ($root->[0] ne $mid) { + my $s = $x->header('Subject'); + my $h = hash_subj($s); + if ($root->[1]->{$h}) { + $s = ''; + } else { + $root->[1]->{$h} = 1; + $s = PublicInbox::Hval->new($s); + $s = $s->as_html; + } + my $m = PublicInbox::Hval->new_msgid($mid); + my $f = PublicInbox::Hval->new($x->header('X-PI-From')); + my $d = PublicInbox::Hval->new($x->header('X-PI-Date')); + $m = $m->as_href . '.html'; + $f = $f->as_html; + $d = $d->as_html . ' UTC'; + if (length($s) == 0) { + $$dst .= "` $f @ $d\n"; + } else { + $$dst .= "` $s\n" . + "$pfx by $f @ $d\n"; + } + } + } + simple_dump($dst, $root, $node->child, $level + 1) if $node->child; + simple_dump($dst, $root, $node->next, $level) if $node->next; +} + +sub hash_subj { + my ($subj) = @_; + $subj =~ s/\A\s+//; + $subj =~ s/\s+\z//; + $subj =~ s/^(?:re|aw):\s*//i; # remove reply prefix (aw: German) + $subj =~ s/\s+/ /; + Digest::SHA::sha1($subj); +} + +sub thread_replies { + my ($dst, $root, $res) = @_; + my @msgs = map { $_->mini_mime } @{$res->{msgs}}; + foreach (@{$res->{msgs}}) { + print STDERR "smsg->path: <", $_->path, ">\n"; + } + require PublicInbox::Thread; + $root->header_set('X-PI-TS', '0'); + my $th = PublicInbox::Thread->new($root, @msgs); + $th->thread; + $th->order(*PublicInbox::Thread::sort_ts); + $root = [ $root->header('Message-ID'), + { hash_subj($root->header('Subject')) => 1 } ]; + simple_dump($dst, $root, $_, 0) for $th->rootset; +} + +sub thread_html_head { + my ($mime) = @_; + my $s = PublicInbox::Hval->new_oneline($mime->header('Subject')); + $s = $s->as_html; + "$s" . PRE_WRAP + +} + +sub thread_entry { + my ($dst, $state, $node, $level) = @_; + # $state = [ $search_res, $seen, undef, 0 (msg_nr) ]; + # $seen is overloaded with 3 types of fields: + # 1) "root_anchor" => anchor_for(Message-ID), + # 2) seen subject hashes: sha1(subject) => 1 + # 3) anchors hashes: "#$sha1_hex" (same as $seen in index_entry) + if (my $mime = $node->message) { + if (length($$dst) == 0) { + $$dst .= thread_html_head($mime); + } + $$dst .= index_entry(undef, $mime, $level, $state); + } + thread_entry($dst, $state, $node->child, $level + 1) if $node->child; + thread_entry($dst, $state, $node->next, $level) if $node->next; +} + +sub load_results { + my ($ctx, $res) = @_; + + require PublicInbox::GitCatFile; + my $git = PublicInbox::GitCatFile->new($ctx->{git_dir}); + my @msgs; + while (my $smsg = shift @{$res->{msgs}}) { + my $m = $smsg->mid; + my $path = mid2path($m); + + # FIXME: duplicated code from Feed.pm + my $mime = eval { + my $str = $git->cat_file("HEAD:$path"); + Email::MIME->new($str); + }; + unless ($@) { + my $t = eval { str2time($mime->header('Date')) }; + defined($t) or $t = 0; + $mime->header_set('X-PI-TS', $t); + push @msgs, $mime; + } + } + \@msgs; } 1; diff --git a/lib/PublicInbox/WWW.pm b/lib/PublicInbox/WWW.pm index 1814286b..bbd438a2 100644 --- a/lib/PublicInbox/WWW.pm +++ b/lib/PublicInbox/WWW.pm @@ -49,9 +49,20 @@ sub run { } elsif ($path_info =~ m!$LISTNAME_RE/f/(\S+)\.html\z!o) { invalid_list_mid(\%ctx, $1, $2) || get_full_html(\%ctx, $cgi); + # thread display + } elsif ($path_info =~ m!$LISTNAME_RE/t/(\S+)\.html\z!o) { + invalid_list_mid(\%ctx, $1, $2) || get_thread(\%ctx, $cgi); + + # subject_path display + } elsif ($path_info =~ m!$LISTNAME_RE/s/(\S+)\.html\z!o) { + my $sp = $2; + invalid_list(\%ctx, $1) || get_subject_path(\%ctx, $cgi, $sp); + # convenience redirects, order matters - } elsif ($path_info =~ m!$LISTNAME_RE/(?:m|f)/(\S+)\z!o) { - invalid_list_mid(\%ctx, $1, $2) || redirect_mid(\%ctx, $cgi); + } elsif ($path_info =~ m!$LISTNAME_RE/(m|f|t|s)/(\S+)\z!o) { + my $pfx = $2; + invalid_list_mid(\%ctx, $1, $3) || + redirect_mid(\%ctx, $cgi, $2); } else { r404(); @@ -130,19 +141,10 @@ sub get_index { # just returns a string ref for the blob in the current ctx sub mid2blob { my ($ctx) = @_; - my $hex = $ctx->{mid}; - my ($x2, $x38) = ($hex =~ /\A([a-f0-9]{2})([a-f0-9]{38})\z/); - - unless (defined $x38) { - # compatibility with old links - require Digest::SHA; - $hex = Digest::SHA::sha1_hex($hex); - ($x2, $x38) = ($hex =~ /\A([a-f0-9]{2})([a-f0-9]{38})\z/); - defined $x38 or die "BUG: not a SHA-1 hex: $hex"; - } - + require PublicInbox::MID; + my $path = PublicInbox::MID::mid2path($ctx->{mid}); my @cmd = ('git', "--git-dir=$ctx->{git_dir}", - qw(cat-file blob), "HEAD:$x2/$x38"); + qw(cat-file blob), "HEAD:$path"); my $cmd = join(' ', @cmd); my $pid = open my $fh, '-|'; defined $pid or die "fork failed: $!\n"; @@ -170,13 +172,13 @@ sub get_mid_html { return r404() unless $x; require PublicInbox::View; - my $mid_href = PublicInbox::Hval::ascii_html( - uri_escape_utf8($ctx->{mid})); - my $pfx = "../f/$mid_href.html"; + my $pfx = msg_pfx($ctx); my $foot = footer($ctx); require Email::MIME; + my $mime = Email::MIME->new($x); + my $srch = searcher($ctx); [ 200, [ 'Content-Type' => 'text/html; charset=UTF-8' ], - [ PublicInbox::View->msg_html(Email::MIME->new($x), $pfx, $foot) ] ]; + [ PublicInbox::View->msg_html($mime, $pfx, $foot, $srch) ] ]; } # /$LISTNAME/f/$MESSAGE_ID.html -> HTML content (fullquotes) @@ -185,10 +187,37 @@ sub get_full_html { my $x = mid2blob($ctx); return r404() unless $x; require PublicInbox::View; + my $foot = footer($ctx); require Email::MIME; + my $mime = Email::MIME->new($x); + my $srch = searcher($ctx); + [ 200, [ 'Content-Type' => 'text/html; charset=UTF-8' ], + [ PublicInbox::View->msg_html($mime, undef, $foot, $srch)] ]; +} + +# /$LISTNAME/t/$MESSAGE_ID.html +sub get_thread { + my ($ctx, $cgi) = @_; + my $srch = searcher($ctx) or return need_search($ctx); + require PublicInbox::View; + my $foot = footer($ctx); + my $body = PublicInbox::View->thread_html($ctx, $foot, $srch) or + return r404(); + [ 200, [ 'Content-Type' => 'text/html; charset=UTF-8' ], + [ $body ] ]; +} + +# /$LISTNAME/s/$SUBJECT_PATH.html +sub get_subject_path { + my ($ctx, $cgi, $sp) = @_; + $ctx->{subject_path} = $sp; + my $srch = searcher($ctx) or return need_search($ctx); + require PublicInbox::View; my $foot = footer($ctx); + my $body = PublicInbox::View->subject_path_html($ctx, $foot, $srch) or + return r404(); [ 200, [ 'Content-Type' => 'text/html; charset=UTF-8' ], - [ PublicInbox::View->msg_html(Email::MIME->new($x), undef, $foot)] ]; + [ $body ] ]; } sub self_url { @@ -202,10 +231,13 @@ sub redirect_list_index { } sub redirect_mid { - my ($ctx, $cgi) = @_; + my ($ctx, $cgi, $pfx) = @_; my $url = self_url($cgi); - $url =~ s!/f/!/m/!; - do_redirect($url . '.html'); + my $anchor = ''; + if (lc($pfx) eq 't') { + $anchor = '#u'; # is used to highlight in View.pm + } + do_redirect($url . ".html$anchor"); } sub do_redirect { @@ -281,4 +313,30 @@ sub footer { ); } +# search support is optional, returns undef if Xapian is not installed +# or not configured for the given GIT_DIR +sub searcher { + my ($ctx) = @_; + eval { + require PublicInbox::Search; + PublicInbox::Search->new($ctx->{git_dir}); + }; +} + +sub need_search { + my ($ctx) = @_; + my $msg = <Search not available for this +public-inbox
Search is not available for this public-inbox
+Return to index
+EOF + [ 501, [ 'Content-Type' => 'text/html; charset=UTF-8' ], [ $msg ] ]; +} + +sub msg_pfx { + my ($ctx) = @_; + my $href = PublicInbox::Hval::ascii_html(uri_escape_utf8($ctx->{mid})); + "../f/$href.html"; +} + 1; diff --git a/public-inbox-index b/public-inbox-index new file mode 100755 index 00000000..9cfcadc2 --- /dev/null +++ b/public-inbox-index @@ -0,0 +1,29 @@ +#!/usr/bin/perl -w +# Copyright (C) 2015, all contributors (git clone git://80x24.org/public-inbox) +# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt) +# Basic tool to create a Xapian search index for a git repository +# configured for public-inbox. +# Usage with libeatmydata +# highly recommended: eatmydata public-inbox-index GIT_DIR + +use strict; +use warnings; +my $usage = "public-inbox-index GIT_DIR"; +use PublicInbox::Config; +use PublicInbox::Search; +use PublicInbox::GitCatFile; + +sub usage { print STDERR "Usage: $usage\n"; exit 1 } +if (@ARGV) { + foreach my $dir (@ARGV) { + index_dir($dir); + } +}; + +sub index_dir { + my ($git_dir) = @_; + -d $git_dir or die "$git_dir does not appear to be a git repository\n"; + my $git = PublicInbox::GitCatFile->new($git_dir); + my $s = PublicInbox::Search->new($git_dir, 1); + $s->index_sync($git); +} diff --git a/public-inbox.cgi b/public-inbox.cgi index 57935c50..ed0f12c8 100755 --- a/public-inbox.cgi +++ b/public-inbox.cgi @@ -29,6 +29,7 @@ if (@ARGV && $ARGV[0] eq 'static') { 301 => 'Moved Permanently', 404 => 'Not Found', 405 => 'Method Not Allowed', + 501 => 'Not Implemented', ); print "Status: $status $codes{$status}\r\n"; diff --git a/t/search.t b/t/search.t new file mode 100644 index 00000000..9bdd3cee --- /dev/null +++ b/t/search.t @@ -0,0 +1,239 @@ +# Copyright (C) 2015, all contributors +# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt) +use strict; +use warnings; +use Test::More; +use File::Temp qw/tempdir/; +use PublicInbox::Search; +use Email::MIME; +use Data::Dumper; +my $tmpdir = tempdir(CLEANUP => 1); +my $git_dir = "$tmpdir/a.git"; +my ($root_id, $last_id); + +is(0, system(qw(git init -q --bare), $git_dir), "git init (main)"); +eval { PublicInbox::Search->new($git_dir) }; +ok($@, "exception raised on non-existent DB"); + +my $rw = PublicInbox::Search->new($git_dir, 1); +my $ro = PublicInbox::Search->new($git_dir); + +{ + my $root = Email::MIME->create( + header_str => [ + Date => 'Fri, 02 Oct 1993 00:00:00 +0000', + Subject => 'Hello world', + 'Message-ID' => '', + From => 'John Smith ', + To => 'list@example.com', + ], + body => "\\m/\n"); + my $last = Email::MIME->create( + header_str => [ + Date => 'Sat, 02 Oct 2010 00:00:00 +0000', + Subject => 'Re: Hello world', + 'In-Reply-To' => '', + 'Message-ID' => '', + From => 'John Smith ', + To => 'list@example.com', + ], + body => "goodbye forever :<\n"); + + my $rv; + $root_id = $rw->add_message($root); + is($root_id, int($root_id), "root_id is an integer: $root_id"); + $last_id = $rw->add_message($last); + is($last_id, int($last_id), "last_id is an integer: $last_id"); +} + +sub filter_mids { + my ($res) = @_; + sort(map { $_->mid } @{$res->{msgs}}); +} + +{ + $ro->reopen; + my $found = $ro->lookup_message(''); + ok($found, "message found"); + is($root_id, $found->{doc_id}, 'doc_id set correctly'); + $found->ensure_metadata; + is($found->mid, 'root@s', 'mid set correctly'); + ok(int($found->thread_id) > 0, 'thread_id is an integer'); + + my @exp = sort qw(root@s last@s); + my $res = $ro->query("path:hello_world"); + my @res = filter_mids($res); + is_deeply(\@res, \@exp, 'got expected results for path: match'); + + foreach my $p (qw(hello hello_ hello_world2 hello_world_)) { + $res = $ro->query("path:$p"); + is($res->{count}, 0, "path variant `$p' does not match"); + } + + $res = $ro->query('subject:(Hello world)'); + @res = filter_mids($res); + is_deeply(\@res, \@exp, 'got expected results for subject:() match'); + + $res = $ro->query('subject:"Hello world"'); + @res = filter_mids($res); + is_deeply(\@res, \@exp, 'got expected results for subject:"" match'); + + $res = $ro->query('subject:"Hello world"', {limit => 1}); + is(scalar @{$res->{msgs}}, 1, "limit works"); + my $first = $res->{msgs}->[0]; + + $res = $ro->query('subject:"Hello world"', {offset => 1}); + is(scalar @{$res->{msgs}}, 1, "offset works"); + my $second = $res->{msgs}->[0]; + + isnt($first, $second, "offset returned different result from limit"); + + foreach my $f (qw(inreplyto references)) { + $res = $ro->query($f . ':root@s'); + @res = filter_mids($res); + is_deeply(\@res, [ 'last@s' ], + "got expected results for $f: match"); + $res = $ro->query($f . ':root'); + is($res->{count}, 0, "no partial mid match"); + } +} + +# ghost vivication +{ + $rw->reopen; + my $rmid = ''; + my $reply_to_ghost = Email::MIME->create( + header_str => [ + Date => 'Sat, 02 Oct 2010 00:00:00 +0000', + Subject => 'Re: ghosts', + 'Message-ID' => '', + 'In-Reply-To' => $rmid, + From => 'Time Traveler ', + To => 'list@example.com', + ], + body => "-_-\n"); + + my $rv; + my $reply_id = $rw->add_message($reply_to_ghost); + is($reply_id, int($reply_id), "reply_id is an integer: $reply_id"); + + my $was_ghost = Email::MIME->create( + header_str => [ + Date => 'Sat, 02 Oct 2010 00:00:01 +0000', + Subject => 'ghosts', + 'Message-ID' => $rmid, + From => 'Laggy Sender ', + To => 'list@example.com', + ], + body => "are real\n"); + + my $ghost_id = $rw->add_message($was_ghost); + is($ghost_id, int($ghost_id), "ghost_id is an integer: $ghost_id"); + ok($ghost_id < $reply_id, "ghost vivified from earlier message"); +} + +# search thread on ghost +{ + $ro->reopen; + + # Subject: + my $res = $ro->query('ghost'); + my @exp = sort qw(ghost-message@s ghost-reply@s); + my @res = filter_mids($res); + is_deeply(\@res, \@exp, 'got expected results for Subject match'); + + # body + $res = $ro->query('goodbye'); + is($res->{msgs}->[0]->mid, 'last@s', 'got goodbye message body'); +} + +# long message-id +{ + $rw->reopen; + $ro->reopen; + my $long_mid = 'last' . ('x' x 60). '@s'; + my $long_midc = Digest::SHA::sha1_hex($long_mid); + + my $long = Email::MIME->create( + header_str => [ + Date => 'Sat, 02 Oct 2010 00:00:00 +0000', + Subject => 'long message ID', + 'References' => ' ', + 'In-Reply-To' => '', + 'Message-ID' => "<$long_mid>", + From => '"Long I.D." ', + To => 'list@example.com', + ], + body => "wut\n"); + my $long_id = $rw->add_message($long); + is($long_id, int($long_id), "long_id is an integer: $long_id"); + + $ro->reopen; + my $res = $ro->query('references:root@s'); + my @res = filter_mids($res); + is_deeply(\@res, [ sort('last@s', $long_midc) ], + "got expected results for references: match"); + + my $replies = $ro->get_replies('root@s'); + $replies = [ filter_mids($replies) ]; + is_deeply($replies, [ filter_mids($res) ], "get_replies matches"); + + my $long_reply_mid = 'reply-to-long@1'; + my $long_reply = Email::MIME->create( + header_str => [ + Subject => 'I break references', + Date => 'Sat, 02 Oct 2010 00:00:00 +0000', + 'Message-ID' => "<$long_reply_mid>", + # No References: + # 'References' => ' <'.$long_mid.'>', + 'In-Reply-To' => "<$long_mid>", + From => '"no1 ', + To => 'list@example.com', + ], + body => "no References\n"); + ok($rw->add_message($long_reply) > $long_id, "inserted long reply"); + + $ro->reopen; + my $t = $ro->get_thread('root@s'); + is($t->{count}, 4, "got all 4 mesages in thread"); + my @exp = sort($long_reply_mid, 'root@s', 'last@s', $long_midc); + @res = filter_mids($t); + is_deeply(\@res, \@exp, "get_thread works"); +} + +# quote prioritization +{ + $rw->reopen; + $rw->add_message(Email::MIME->create( + header_str => [ + Date => 'Sat, 02 Oct 2010 00:00:01 +0000', + Subject => 'Hello', + 'Message-ID' => '', + From => 'Quoter ', + To => 'list@example.com', + ], + body => "> theatre illusions\nfade\n")); + + $rw->add_message(Email::MIME->create( + header_str => [ + Date => 'Sat, 02 Oct 2010 00:00:02 +0000', + Subject => 'Hello', + 'Message-ID' => '', + From => 'Non-Quoter', + To => 'list@example.com', + ], + body => "theatre\nfade\n")); + my $res = $rw->query("theatre"); + is($res->{count}, 2, "got both matches"); + is($res->{msgs}->[0]->mid, 'nquote@a', "non-quoted scores higher"); + is($res->{msgs}->[1]->mid, 'quote@a', "quoted result still returned"); + + $res = $rw->query("illusions"); + is($res->{count}, 1, "got a match for quoted text"); + is($res->{msgs}->[0]->mid, 'quote@a', + "quoted result returned if nothing else"); +} + +done_testing(); + +1;