X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FExtMsg.pm;h=49fc1ac5521c1506acb137afaf1f667f666f231d;hb=55b707d788ce13696e4411389583e720ea6dab01;hp=955ada7ee8a6923281c9b3954804cabc18a44e78;hpb=8888a03c12dcd0f17ff611f4617523942a3aed49;p=public-inbox.git

diff --git a/lib/PublicInbox/ExtMsg.pm b/lib/PublicInbox/ExtMsg.pm
index 955ada7e..49fc1ac5 100644
--- a/lib/PublicInbox/ExtMsg.pm
+++ b/lib/PublicInbox/ExtMsg.pm
@@ -1,5 +1,5 @@
-# Copyright (C) 2015 all contributors <meta@public-inbox.org>
-# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
+# Copyright (C) 2015-2019 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 #
 # Used by the web interface to link to messages outside of the our
 # public-inboxes.  Mail threads may cross projects/threads; so
@@ -8,105 +8,136 @@
 package PublicInbox::ExtMsg;
 use strict;
 use warnings;
-use PublicInbox::Hval;
-use PublicInbox::MID qw/mid2path/;
+use PublicInbox::Hval qw(ascii_html prurl);
 use PublicInbox::WwwStream;
+our $MIN_PARTIAL_LEN = 16;
 
 # TODO: user-configurable
-our @EXT_URL = (
-	'http://mid.gmane.org/%s',
-	'https://lists.debian.org/msgid-search/%s',
+our @EXT_URL = map { ascii_html($_) } (
 	# leading "//" denotes protocol-relative (http:// or https://)
-	'//mid.mail-archive.com/%s',
 	'//marc.info/?i=%s',
+	'//www.mail-archive.com/search?l=mid&q=%s',
+	'nntp://news.gmane.org/%s',
+	'https://lists.debian.org/msgid-search/%s',
+	'//docs.FreeBSD.org/cgi/mid.cgi?db=mid&id=%s',
+	'https://www.w3.org/mid/%s',
+	'http://www.postgresql.org/message-id/%s',
+	'https://lists.debconf.org/cgi-lurker/keyword.cgi?'.
+		'doc-url=/lurker&format=en.html&query=id:%s'
 );
 
-sub ext_msg {
-	my ($ctx) = @_;
-	my $cur = $ctx->{-inbox};
-	my $mid = $ctx->{mid};
-
-	eval { require PublicInbox::Search };
-	my $have_xap = $@ ? 0 : 1;
-	my (@nox, @ibx, @found);
+sub PARTIAL_MAX () { 100 }
 
-	$ctx->{www}->{pi_config}->each_inbox(sub {
-		my ($other) = @_;
-		return if $other->{name} eq $cur->{name} || !$other->base_url;
+sub mids_from_mset { # Search::retry_reopen callback
+	[ map { PublicInbox::SearchMsg::from_mitem($_)->mid } $_[0]->items ];
+}
 
-		my $s = $other->search;
-		if (!$s) {
-			push @nox, $other;
-			return;
+sub search_partial ($$) {
+	my ($srch, $mid) = @_;
+	return if length($mid) < $MIN_PARTIAL_LEN;
+	my $opt = { limit => PARTIAL_MAX, mset => 2 };
+	my @try = ("m:$mid*");
+	my $chop = $mid;
+	if ($chop =~ s/(\W+)(\w*)\z//) {
+		my ($delim, $word) = ($1, $2);
+		if (length($word)) {
+			push @try, "m:$chop$delim";
+			push @try, "m:$chop$delim*";
 		}
+		push @try, "m:$chop";
+		push @try, "m:$chop*";
+	}
 
-		# try to find the URL with Xapian to avoid forking
-		my $doc_id = eval { $s->find_unique_doc_id('mid', $mid) };
-		if ($@) {
-			# xapian not configured properly for this repo
-			push @nox, $other;
-			return;
+	# break out long words individually to search for, because
+	# too many messages begin with "Pine.LNX." (or "alpine" or "nycvar")
+	if ($mid =~ /\w{9,}/) {
+		my @long = ($mid =~ m!(\w{3,})!g);
+		push(@try, join(' ', map { "m:$_" } @long));
+
+		# is the last element long enough to not trigger excessive
+		# wildcard matches?
+		if (length($long[-1]) > 8) {
+			$long[-1] .= '*';
+			push(@try, join(' ', map { "m:$_" } @long));
 		}
+	}
 
-		# maybe we found it!
-		if (defined $doc_id) {
-			push @found, $other;
-		} else {
-			# no point in trying the fork fallback if we
-			# know Xapian is up-to-date but missing the
-			# message in the current repo
-			push @ibx, $other;
-		}
-	});
+	foreach my $m (@try) {
+		# If Xapian can't handle the wildcard since it
+		# has too many results.  $@ can be
+		# Search::Xapian::QueryParserError or even:
+		# "something terrible happened at ../Search/Xapian/Enquire.pm"
+		my $mset = eval { $srch->query($m, $opt) } or next;
+		my $mids = $srch->retry_reopen(\&mids_from_mset, $mset);
+		return $mids if scalar(@$mids);
+	}
+}
 
-	return exact($ctx, \@found, $mid) if @found;
+sub ext_msg_i {
+	my ($other, $arg) = @_;
+	my ($cur, $mid, $ibxs, $found) = @$arg;
 
-	# Xapian not installed or configured for some repos,
-	# do a full MID check (this is expensive...):
-	if (@nox) {
-		my $path = mid2path($mid);
-		foreach my $other (@nox) {
-			my (undef, $type, undef) = $other->path_check($path);
+	return if $other->{name} eq $cur->{name} || !$other->base_url;
 
-			if ($type && $type eq 'blob') {
-				push @found, $other;
-			}
-		}
+	my $mm = $other->mm or return;
+
+	# try to find the URL with Msgmap to avoid forking
+	my $num = $mm->num_for($mid);
+	if (defined $num) {
+		push @$found, $other;
+	} else {
+		# no point in trying the fork fallback if we
+		# know Xapian is up-to-date but missing the
+		# message in the current repo
+		push @$ibxs, $other;
 	}
-	return exact($ctx, \@found, $mid) if @found;
+}
+
+sub ext_msg {
+	my ($ctx) = @_;
+	my $cur = $ctx->{-inbox};
+	my $mid = $ctx->{mid};
+
+	eval { require PublicInbox::Msgmap };
+	my $ibxs = [];
+	my $found = [];
+	my $arg = [ $cur, $mid, $ibxs, $found ];
+
+	$ctx->{www}->{pi_config}->each_inbox(\&ext_msg_i, $arg);
+
+	return exact($ctx, $found, $mid) if @$found;
 
 	# fall back to partial MID matching
-	my $n_partial = 0;
 	my @partial;
+	my $n_partial = 0;
+	my $srch = $cur->search;
+	my $mids = search_partial($srch, $mid) if $srch;
+	if ($mids) {
+		$n_partial = scalar(@$mids);
+		push @partial, [ $cur, $mids ];
+	}
 
-	eval { require PublicInbox::Msgmap };
-	my $have_mm = $@ ? 0 : 1;
-	if ($have_mm) {
-		my $tmp_mid = $mid;
-again:
-		unshift @ibx, $cur;
-		foreach my $ibx (@ibx) {
-			my $mm = $ibx->mm or next;
-			if (my $res = $mm->mid_prefixes($tmp_mid)) {
-				$n_partial += scalar(@$res);
-				push @partial, [ $ibx, $res ];
-			}
-		}
-		# fixup common errors:
-		if (!$n_partial && $tmp_mid =~ s,/[tTf],,) {
-			goto again;
+	# can't find a partial match in current inbox, try the others:
+	if (!$n_partial && length($mid) >= $MIN_PARTIAL_LEN) {
+		foreach my $ibx (@$ibxs) {
+			$srch = $ibx->search or next;
+			$mids = search_partial($srch, $mid) or next;
+			$n_partial += scalar(@$mids);
+			push @partial, [ $ibx, $mids];
+			last if $n_partial >= PARTIAL_MAX;
 		}
 	}
 
 	my $code = 404;
 	my $h = PublicInbox::Hval->new_msgid($mid);
-	my $href = $h->as_href;
+	my $href = $h->{href};
 	my $html = $h->as_html;
 	my $title = "&lt;$html&gt; not found";
 	my $s = "<pre>Message-ID &lt;$html&gt;\nnot found\n";
 	if ($n_partial) {
 		$code = 300;
 		my $es = $n_partial == 1 ? '' : 'es';
+		$n_partial .= '+' if ($n_partial == PARTIAL_MAX);
 		$s .= "\n$n_partial partial match$es found:\n\n";
 		my $cur_name = $cur->{name};
 		foreach my $pair (@partial) {
@@ -115,7 +146,7 @@ again:
 			my $u = $ibx->base_url($env) or next;
 			foreach my $m (@$res) {
 				my $p = PublicInbox::Hval->new_msgid($m);
-				my $r = $p->as_href;
+				my $r = $p->{href};
 				my $t = $p->as_html;
 				$s .= qq{<a\nhref="$u$r/">$u$t/</a>\n};
 			}
@@ -140,7 +171,7 @@ sub ext_urls {
 		my $env = $ctx->{env};
 		my $e = "\nPerhaps try an external site:\n\n";
 		foreach my $url (@EXT_URL) {
-			my $u = PublicInbox::Hval::prurl($env, $url);
+			my $u = prurl($env, $url);
 			my $r = sprintf($u, $href);
 			my $t = sprintf($u, $html);
 			$e .= qq{<a\nhref="$r">$t</a>\n};
@@ -153,7 +184,7 @@ sub ext_urls {
 sub exact {
 	my ($ctx, $found, $mid) = @_;
 	my $h = PublicInbox::Hval->new_msgid($mid);
-	my $href = $h->as_href;
+	my $href = $h->{href};
 	my $html = $h->as_html;
 	my $title = "&lt;$html&gt; found in ";
 	my $end = @$found == 1 ? 'another inbox' : 'other inboxes';