X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FExtMsg.pm;h=49fc1ac5521c1506acb137afaf1f667f666f231d;hb=55b707d788ce13696e4411389583e720ea6dab01;hp=c71510f548191b966cfb4835256764f4ceeae7f3;hpb=471cc731579002419d23f64509497dd486dd632c;p=public-inbox.git diff --git a/lib/PublicInbox/ExtMsg.pm b/lib/PublicInbox/ExtMsg.pm index c71510f5..49fc1ac5 100644 --- a/lib/PublicInbox/ExtMsg.pm +++ b/lib/PublicInbox/ExtMsg.pm @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2018 all contributors +# Copyright (C) 2015-2019 all contributors # License: AGPL-3.0+ # # Used by the web interface to link to messages outside of the our @@ -8,16 +8,16 @@ package PublicInbox::ExtMsg; use strict; use warnings; -use PublicInbox::Hval; -use PublicInbox::MID qw/mid2path/; +use PublicInbox::Hval qw(ascii_html prurl); use PublicInbox::WwwStream; +our $MIN_PARTIAL_LEN = 16; # TODO: user-configurable -our @EXT_URL = ( +our @EXT_URL = map { ascii_html($_) } ( # leading "//" denotes protocol-relative (http:// or https://) '//marc.info/?i=%s', '//www.mail-archive.com/search?l=mid&q=%s', - 'http://mid.gmane.org/%s', + 'nntp://news.gmane.org/%s', 'https://lists.debian.org/msgid-search/%s', '//docs.FreeBSD.org/cgi/mid.cgi?db=mid&id=%s', 'https://www.w3.org/mid/%s', @@ -26,96 +26,105 @@ our @EXT_URL = ( 'doc-url=/lurker&format=en.html&query=id:%s' ); -sub ext_msg { - my ($ctx) = @_; - my $cur = $ctx->{-inbox}; - my $mid = $ctx->{mid}; +sub PARTIAL_MAX () { 100 } - eval { require PublicInbox::Search }; - my $have_xap = $@ ? 0 : 1; - my (@nox, @ibx, @found); - - $ctx->{www}->{pi_config}->each_inbox(sub { - my ($other) = @_; - return if $other->{name} eq $cur->{name} || !$other->base_url; +sub mids_from_mset { # Search::retry_reopen callback + [ map { PublicInbox::SearchMsg::from_mitem($_)->mid } $_[0]->items ]; +} - my $s = $other->search; - if (!$s) { - push @nox, $other; - return; +sub search_partial ($$) { + my ($srch, $mid) = @_; + return if length($mid) < $MIN_PARTIAL_LEN; + my $opt = { limit => PARTIAL_MAX, mset => 2 }; + my @try = ("m:$mid*"); + my $chop = $mid; + if ($chop =~ s/(\W+)(\w*)\z//) { + my ($delim, $word) = ($1, $2); + if (length($word)) { + push @try, "m:$chop$delim"; + push @try, "m:$chop$delim*"; } + push @try, "m:$chop"; + push @try, "m:$chop*"; + } - # try to find the URL with Xapian to avoid forking - my $doc_id = eval { $s->find_first_doc_id('Q' . $mid) }; - if ($@) { - # xapian not configured properly for this repo - push @nox, $other; - return; + # break out long words individually to search for, because + # too many messages begin with "Pine.LNX." (or "alpine" or "nycvar") + if ($mid =~ /\w{9,}/) { + my @long = ($mid =~ m!(\w{3,})!g); + push(@try, join(' ', map { "m:$_" } @long)); + + # is the last element long enough to not trigger excessive + # wildcard matches? + if (length($long[-1]) > 8) { + $long[-1] .= '*'; + push(@try, join(' ', map { "m:$_" } @long)); } + } - # maybe we found it! - if (defined $doc_id) { - push @found, $other; - } else { - # no point in trying the fork fallback if we - # know Xapian is up-to-date but missing the - # message in the current repo - push @ibx, $other; - } - }); + foreach my $m (@try) { + # If Xapian can't handle the wildcard since it + # has too many results. $@ can be + # Search::Xapian::QueryParserError or even: + # "something terrible happened at ../Search/Xapian/Enquire.pm" + my $mset = eval { $srch->query($m, $opt) } or next; + my $mids = $srch->retry_reopen(\&mids_from_mset, $mset); + return $mids if scalar(@$mids); + } +} - return exact($ctx, \@found, $mid) if @found; +sub ext_msg_i { + my ($other, $arg) = @_; + my ($cur, $mid, $ibxs, $found) = @$arg; - # Xapian not installed or configured for some repos, - # do a full MID check (this is expensive...): - if (@nox) { - my $path = mid2path($mid); - foreach my $other (@nox) { - my (undef, $type, undef) = $other->path_check($path); + return if $other->{name} eq $cur->{name} || !$other->base_url; - if ($type && $type eq 'blob') { - push @found, $other; - } - } + my $mm = $other->mm or return; + + # try to find the URL with Msgmap to avoid forking + my $num = $mm->num_for($mid); + if (defined $num) { + push @$found, $other; + } else { + # no point in trying the fork fallback if we + # know Xapian is up-to-date but missing the + # message in the current repo + push @$ibxs, $other; } - return exact($ctx, \@found, $mid) if @found; +} + +sub ext_msg { + my ($ctx) = @_; + my $cur = $ctx->{-inbox}; + my $mid = $ctx->{mid}; + + eval { require PublicInbox::Msgmap }; + my $ibxs = []; + my $found = []; + my $arg = [ $cur, $mid, $ibxs, $found ]; + + $ctx->{www}->{pi_config}->each_inbox(\&ext_msg_i, $arg); + + return exact($ctx, $found, $mid) if @$found; # fall back to partial MID matching - my $n_partial = 0; my @partial; - - if (my $mm = $cur->mm) { - my $tmp_mid = $mid; - my $res = $mm->mid_prefixes($tmp_mid, 100); - if ($res && scalar(@$res)) { - $n_partial += scalar(@$res); - push @partial, [ $cur, $res ]; - # fixup common errors: - } elsif ($tmp_mid =~ s,/[tTf],,) { - $res = $mm->mid_prefixes($tmp_mid, 100); - if ($res && scalar(@$res)) { - $n_partial += scalar(@$res); - push @partial, [ $cur, $res ]; - } - } + my $n_partial = 0; + my $srch = $cur->search; + my $mids = search_partial($srch, $mid) if $srch; + if ($mids) { + $n_partial = scalar(@$mids); + push @partial, [ $cur, $mids ]; } # can't find a partial match in current inbox, try the others: - if (!$n_partial && length($mid) >= 16) { - my $tmp_mid = $mid; -again: - foreach my $ibx (@ibx) { - my $mm = $ibx->mm or next; - my $res = $mm->mid_prefixes($tmp_mid, 100); - if ($res && scalar(@$res)) { - $n_partial += scalar(@$res); - push @partial, [ $ibx, $res ]; - last if $n_partial >= 100; - } - } - # fixup common errors: - if (!$n_partial && $tmp_mid =~ s,/[tTf],,) { - goto again; + if (!$n_partial && length($mid) >= $MIN_PARTIAL_LEN) { + foreach my $ibx (@$ibxs) { + $srch = $ibx->search or next; + $mids = search_partial($srch, $mid) or next; + $n_partial += scalar(@$mids); + push @partial, [ $ibx, $mids]; + last if $n_partial >= PARTIAL_MAX; } } @@ -128,6 +137,7 @@ again: if ($n_partial) { $code = 300; my $es = $n_partial == 1 ? '' : 'es'; + $n_partial .= '+' if ($n_partial == PARTIAL_MAX); $s .= "\n$n_partial partial match$es found:\n\n"; my $cur_name = $cur->{name}; foreach my $pair (@partial) { @@ -161,7 +171,7 @@ sub ext_urls { my $env = $ctx->{env}; my $e = "\nPerhaps try an external site:\n\n"; foreach my $url (@EXT_URL) { - my $u = PublicInbox::Hval::prurl($env, $url); + my $u = prurl($env, $url); my $r = sprintf($u, $href); my $t = sprintf($u, $html); $e .= qq{$t\n};