search: support searching on List-Id

[public-inbox.git] / lib / PublicInbox / Search.pm

diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm
index eb1a1446b6f10f4552431f99253d87f2608f06be..b7db2b9f7fcc91aacc317ef3e439b6349934083c 100644 (file)
--- a/lib/PublicInbox/Search.pm
+++ b/lib/PublicInbox/Search.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2019 all contributors <meta@public-inbox.org>
+# Copyright (C) 2015-2020 all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 # based on notmuch, but with no concept of folders, files or flags
 #
@@ -12,9 +12,7 @@ use constant TS => 0;  # Received: header in Unix time
 use constant YYYYMMDD => 1; # Date: header for searching in the WWW UI
 use constant DT => 2; # Date: YYYYMMDDHHMMSS
 
-use PublicInbox::SearchMsg;
-use PublicInbox::MIME;
-use PublicInbox::MID qw/id_compress/;
+use PublicInbox::Smsg;
 use PublicInbox::Over;
 my $QP_FLAGS;
 our %X = map { $_ => 0 } qw(BoolWeight Database Enquire
@@ -38,8 +36,8 @@ sub load_xapian () {
                $ENQ_ASCENDING = $x eq 'Xapian' ?
                                1 : Search::Xapian::ENQ_ASCENDING();
 
-               # for SearchMsg:
-               *PublicInbox::SearchMsg::sortable_unserialise =
+               # for Smsg:
+               *PublicInbox::Smsg::sortable_unserialise =
                                                $Xap.'::sortable_unserialise';
                # n.b. FLAG_PURE_NOT is expensive not suitable for a public
                # website as it could become a denial-of-service vector
@@ -49,7 +47,7 @@ sub load_xapian () {
                # or make indexlevel=medium as default
                $QP_FLAGS = FLAG_PHRASE() | FLAG_BOOLEAN() | FLAG_LOVEHATE() |
                                FLAG_WILDCARD();
-               last;
+               return 1;
        }
        undef;
 }
@@ -79,11 +77,17 @@ use constant {
        # 15 - see public-inbox-v2-format(5)
        #      further bumps likely unnecessary, we'll suggest in-place
        #      "--reindex" use for further fixes and tweaks
+       #
+       #      public-inbox v1.5.0 adds (still SCHEMA_VERSION=15):
+       #      * "lid:" and "l:" for List-Id searches
        SCHEMA_VERSION => 15,
 };
 
+# note: the non-X term prefix allocations are shared with
+# Xapian omega, see xapian-applications/omega/docs/termprefixes.rst
 my %bool_pfx_external = (
        mid => 'Q', # Message-ID (full/exact), this is mostly uniQue
+       lid => 'G', # newsGroup (or similar entity), just inside <>
        dfpre => 'XDFPRE',
        dfpost => 'XDFPOST',
        dfblob => 'XDFPRE XDFPOST',
@@ -94,6 +98,7 @@ my %prob_prefix = (
        # for mairix compatibility
        s => 'S',
        m => 'XM', # 'mid:' (bool) is exact, 'm:' (prob) can do partial
+       l => 'XL', # 'lid:' (bool) is exact, 'l:' (prob) can do partial
        f => 'A',
        t => 'XTO',
        tc => 'XTO XCC',
@@ -136,6 +141,8 @@ EOF
        'f:' => 'match within the From header',
        'a:' => 'match within the To, Cc, and From headers',
        'tc:' => 'match within the To and Cc headers',
+       'lid:' => 'exact contents of the List-Id',
+       'l:' => 'partial match contents of the List-Id header',
        'bs:' => 'match within the Subject and body',
        'dfn:' => 'match filename from diff',
        'dfa:' => 'match diff removed (-) lines',
@@ -150,7 +157,7 @@ chomp @HELP;
 
 sub xdir ($;$) {
        my ($self, $rdonly) = @_;
-       if ($self->{version} == 1) {
+       if ($self->{ibx_ver} == 1) {
                "$self->{inboxdir}/public-inbox/xapian" . SCHEMA_VERSION;
        } else {
                my $dir = "$self->{inboxdir}/xap" . SCHEMA_VERSION;
@@ -167,7 +174,7 @@ sub _xdb ($) {
        my $dir = xdir($self, 1);
        my ($xdb, $slow_phrase);
        my $qpf = \($self->{qp_flags} ||= $QP_FLAGS);
-       if ($self->{version} >= 2) {
+       if ($self->{ibx_ver} >= 2) {
                foreach my $shard (<$dir/*>) {
                        -d $shard && $shard =~ m!/[0-9]+\z! or next;
                        my $sub = $X{Database}->new($shard);
@@ -200,7 +207,7 @@ sub new {
        my $self = bless {
                inboxdir => $ibx->{inboxdir},
                altid => $ibx->{altid},
-               version => $ibx->{version} // 1,
+               ibx_ver => $ibx->version,
        }, $class;
        my $dir = xdir($self, 1);
        $self->{over_ro} = PublicInbox::Over->new("$dir/over.sqlite3");
@@ -281,7 +288,7 @@ sub _enquire_once { # retry_reopen callback
        my $limit = $opts->{limit} || 50;
        my $mset = $enquire->get_mset($offset, $limit);
        return $mset if $opts->{mset};
-       my @msgs = map { PublicInbox::SearchMsg::from_mitem($_) } $mset->items;
+       my @msgs = map { PublicInbox::Smsg::from_mitem($_) } $mset->items;
        return \@msgs unless wantarray;
 
        ($mset->get_matches_estimated, \@msgs)
@@ -315,9 +322,11 @@ sub qp {
        # we do not actually create AltId objects,
        # just parse the spec to avoid the extra DB handles for now.
        if (my $altid = $self->{altid}) {
-               my $user_pfx = $self->{-user_pfx} ||= [];
+               my $user_pfx = $self->{-user_pfx} = [];
                for (@$altid) {
                        # $_ = 'serial:gmane:/path/to/gmane.msgmap.sqlite3'
+                       # note: Xapian supports multibyte UTF-8, /^[0-9]+$/,
+                       # and '_' with prefixes matching \w+
                        /\Aserial:(\w+):/ or next;
                        my $pfx = $1;
                        push @$user_pfx, "$pfx:", <<EOF;
@@ -336,11 +345,6 @@ EOF
        $self->{query_parser} = $qp;
 }
 
-sub lookup_article {
-       my ($self, $num) = @_;
-       $self->{over_ro}->get_art($num);
-}
-
 sub help {
        my ($self) = @_;
        $self->qp; # parse altids