X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FExtMsg.pm;h=51e7799de8640929be9538713d3f2ee7e1b1081d;hb=a46893a2b5dabfdbcf7b593ac19967daecfb1772;hp=1c0887cd616e89ba4511f46d50e0e58612efcea6;hpb=b872759b28cd4726b9f16f5214d042fdc50b1e93;p=public-inbox.git diff --git a/lib/PublicInbox/ExtMsg.pm b/lib/PublicInbox/ExtMsg.pm index 1c0887cd..51e7799d 100644 --- a/lib/PublicInbox/ExtMsg.pm +++ b/lib/PublicInbox/ExtMsg.pm @@ -1,92 +1,199 @@ -# Copyright (C) 2015 all contributors -# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt) +# Copyright (C) 2015-2018 all contributors +# License: AGPL-3.0+ +# +# Used by the web interface to link to messages outside of the our +# public-inboxes. Mail threads may cross projects/threads; so +# we should ensure users can find more easily find them on other +# sites. package PublicInbox::ExtMsg; use strict; use warnings; -use URI::Escape qw(uri_escape_utf8); use PublicInbox::Hval; -use PublicInbox::MID qw/mid_compress mid2path/; +use PublicInbox::MID qw/mid2path/; +use PublicInbox::WwwStream; + +# TODO: user-configurable +our @EXT_URL = ( + # leading "//" denotes protocol-relative (http:// or https://) + '//marc.info/?i=%s', + '//www.mail-archive.com/search?l=mid&q=%s', + 'nntp://news.gmane.org/%s', + 'https://lists.debian.org/msgid-search/%s', + '//docs.FreeBSD.org/cgi/mid.cgi?db=mid&id=%s', + 'https://www.w3.org/mid/%s', + 'http://www.postgresql.org/message-id/%s', + 'https://lists.debconf.org/cgi-lurker/keyword.cgi?'. + 'doc-url=/lurker&format=en.html&query=id:%s' +); + +sub PARTIAL_MAX () { 100 } + +sub search_partial ($$) { + my ($srch, $mid) = @_; + my $opt = { limit => PARTIAL_MAX, mset => 2 }; + my @try = ("m:$mid*"); + my $chop = $mid; + if ($chop =~ s/(\W+)(\w*)\z//) { + my ($delim, $word) = ($1, $2); + if (length($word)) { + push @try, "m:$chop$delim"; + push @try, "m:$chop$delim*"; + } + push @try, "m:$chop"; + push @try, "m:$chop*"; + } + + # break out long words individually to search for, because + # too many messages begin with "Pine.LNX." (or "alpine" or "nycvar") + if ($mid =~ /\w{9,}/) { + my @long = ($mid =~ m!(\w{3,})!g); + push(@try, join(' ', map { "m:$_" } @long)); + + # is the last element long enough to not trigger excessive + # wildcard matches? + if (length($long[-1]) > 8) { + $long[-1] .= '*'; + push(@try, join(' ', map { "m:$_" } @long)); + } + } + + foreach my $m (@try) { + my $mset = eval { $srch->query($m, $opt) }; + if (ref($@) eq 'Search::Xapian::QueryParserError') { + # If Xapian can't handle the wildcard since it + # has too many results. + next; + } + my @mids = map { + my $doc = $_->get_document; + PublicInbox::SearchMsg->load_doc($doc)->mid; + } $mset->items; + return \@mids if scalar(@mids); + } +} sub ext_msg { my ($ctx) = @_; - my $pi_config = $ctx->{pi_config}; - my $listname = $ctx->{listname}; + my $cur = $ctx->{-inbox}; my $mid = $ctx->{mid}; - my $cmid = mid_compress($mid); - - eval { require PublicInbox::Search }; - my $have_xap = $@ ? 0 : 1; - my @nox; - - foreach my $k (keys %$pi_config) { - $k =~ /\Apublicinbox\.([A-Z0-9a-z-]+)\.url\z/ or next; - my $list = $1; - next if $list eq $listname; - - my $git_dir = $pi_config->{"publicinbox.$list.mainrepo"}; - defined $git_dir or next; - - my $url = $pi_config->{"publicinbox.$list.url"}; - defined $url or next; - - $url =~ s!/+\z!!; - - # try to find the URL with Xapian to avoid forking - if ($have_xap) { - my $doc_id = eval { - my $s = PublicInbox::Search->new($git_dir); - $s->find_unique_doc_id('mid', $cmid); - }; - if ($@) { - # xapian not configured for this repo - } else { - # maybe we found it! - return r302($url, $cmid) if (defined $doc_id); - - # no point in trying the fork fallback if we - # know Xapian is up-to-date but missing the - # message in the current repo - next; - } - } - # queue up for forking after we've tried Xapian on all of them - push @nox, { git_dir => $git_dir, url => $url }; - } + eval { require PublicInbox::Msgmap }; + my $have_mm = $@ ? 0 : 1; + my (@ibx, @found); - # Xapian not installed or configured for some repos - my $path = "HEAD:" . mid2path($cmid); + $ctx->{www}->{pi_config}->each_inbox(sub { + my ($other) = @_; + return if $other->{name} eq $cur->{name} || !$other->base_url; - foreach my $n (@nox) { - my @cmd = ('git', "--git-dir=$n->{git_dir}", 'cat-file', - '-t', $path); - my $pid = open my $fh, '-|'; - defined $pid or die "fork failed: $!\n"; + my $mm = $other->mm or return; - if ($pid == 0) { - open STDERR, '>', '/dev/null'; # ignore errors - exec @cmd or die "exec failed: $!\n"; + # try to find the URL with Msgmap to avoid forking + my $num = $mm->num_for($mid); + if (defined $num) { + push @found, $other; } else { - my $type = eval { local $/; <$fh> }; - close $fh; - if ($? == 0 && $type eq "blob\n") { - return r302($n->{url}, $cmid); - } + # no point in trying the fork fallback if we + # know Xapian is up-to-date but missing the + # message in the current repo + push @ibx, $other; } + }); + + return exact($ctx, \@found, $mid) if @found; + + # fall back to partial MID matching + my @partial; + my $n_partial = 0; + my $srch = $cur->search; + my $mids = search_partial($srch, $mid) if $srch; + if ($mids) { + $n_partial = scalar(@$mids); + push @partial, [ $cur, $mids ]; } - # Fall back to external repos + # can't find a partial match in current inbox, try the others: + if (!$n_partial && length($mid) >= 16) { + foreach my $ibx (@ibx) { + $srch = $ibx->search or next; + $mids = search_partial($srch, $mid) or next; + $n_partial += scalar(@$mids); + push @partial, [ $ibx, $mids]; + last if $n_partial >= PARTIAL_MAX; + } + } - [404, ['Content-Type'=>'text/plain'], ['Not found']]; + my $code = 404; + my $h = PublicInbox::Hval->new_msgid($mid); + my $href = $h->{href}; + my $html = $h->as_html; + my $title = "<$html> not found"; + my $s = "
Message-ID <$html>\nnot found\n";
+	if ($n_partial) {
+		$code = 300;
+		my $es = $n_partial == 1 ? '' : 'es';
+		$n_partial .= '+' if ($n_partial == PARTIAL_MAX);
+		$s .= "\n$n_partial partial match$es found:\n\n";
+		my $cur_name = $cur->{name};
+		foreach my $pair (@partial) {
+			my ($ibx, $res) = @$pair;
+			my $env = $ctx->{env} if $ibx->{name} eq $cur_name;
+			my $u = $ibx->base_url($env) or next;
+			foreach my $m (@$res) {
+				my $p = PublicInbox::Hval->new_msgid($m);
+				my $r = $p->{href};
+				my $t = $p->as_html;
+				$s .= qq{$u$t/\n};
+			}
+		}
+	}
+	my $ext = ext_urls($ctx, $mid, $href, $html);
+	if ($ext ne '') {
+		$s .= $ext;
+		$code = 300;
+	}
+	$ctx->{-html_tip} = $s .= '
'; + $ctx->{-title_html} = $title; + $ctx->{-upfx} = '../'; + PublicInbox::WwwStream->response($ctx, $code); +} + +sub ext_urls { + my ($ctx, $mid, $href, $html) = @_; + + # Fall back to external repos if configured + if (@EXT_URL && index($mid, '@') >= 0) { + my $env = $ctx->{env}; + my $e = "\nPerhaps try an external site:\n\n"; + foreach my $url (@EXT_URL) { + my $u = PublicInbox::Hval::prurl($env, $url); + my $r = sprintf($u, $href); + my $t = sprintf($u, $html); + $e .= qq{$t\n}; + } + return $e; + } + '' } -# Redirect to another public-inbox which is mapped by $pi_config -sub r302 { - my ($url, $mid) = @_; - $url .= '/' . uri_escape_utf8($mid) . '/'; - [ 302, - [ 'Location' => $url, 'Content-Type' => 'text/plain' ], - [ "Redirecting to\n$url\n" ] ] +sub exact { + my ($ctx, $found, $mid) = @_; + my $h = PublicInbox::Hval->new_msgid($mid); + my $href = $h->{href}; + my $html = $h->as_html; + my $title = "<$html> found in "; + my $end = @$found == 1 ? 'another inbox' : 'other inboxes'; + $ctx->{-title_html} = $title . $end; + $ctx->{-upfx} = '../'; + my $ext_urls = ext_urls($ctx, $mid, $href, $html); + my $code = (@$found == 1 && $ext_urls eq '') ? 200 : 300; + $ctx->{-html_tip} = join('', + "
Message-ID: <$html>\nfound in $end:\n\n",
+				(map {
+					my $u = $_->base_url;
+					qq($u$html/\n)
+				} @$found),
+			$ext_urls, '
'); + PublicInbox::WwwStream->response($ctx, $code); } 1;