searchidx: index THREADID in Xapian

This is the `tid' column from over.sqlite3; and will be used for
IMAP and JMAP search (among other things).

diff --git a/Documentation/standards.perl b/Documentation/standards.perl
index a64f033e63c13347b9fe5e1790966fcb4041f3c2..0ac6cc525efea8ef10acf28b4000a96b9db24219 100755 (executable)
--- a/Documentation/standards.perl
+++ b/Documentation/standards.perl
@@ -48,8 +48,12 @@ my $rfcs = [
        # 5032 = 'WITHIN search extension for IMAP',
        4978 => 'IMAP COMPRESS Extension',
        # 5182 = 'IMAP Extension for Referencing the Last SEARCH Result',
+       # 5256 => 'IMAP SORT and THREAD extensions',
        # 5738 =>  'IMAP Support for UTF-8',
        # 8474 => 'IMAP Extension for Object Identifiers',
+
+       # 8620 => JSON Meta Application Protocol (JMAP)
+       # 8621 => JSON Meta Application Protocol (JMAP) for Mail
        # ...
 
        # TODO: flesh this out

diff --git a/lib/PublicInbox/Over.pm b/lib/PublicInbox/Over.pm
index a055b4cdbe6114061f3e20d3dfb782418521b304..34d0b05da4188dd5267795da6ee95faf7bed9890 100644 (file)
--- a/lib/PublicInbox/Over.pm
+++ b/lib/PublicInbox/Over.pm
@@ -213,7 +213,7 @@ sub get_art {
        my ($self, $num) = @_;
        # caching $sth ourselves is faster than prepare_cached
        my $sth = $self->{-get_art} //= $self->connect->prepare(<<'');
-SELECT num,ds,ts,ddd FROM over WHERE num = ? LIMIT 1
+SELECT num,tid,ds,ts,ddd FROM over WHERE num = ? LIMIT 1
 
        $sth->execute($num);
        my $smsg = $sth->fetchrow_hashref;

diff --git a/lib/PublicInbox/OverIdx.pm b/lib/PublicInbox/OverIdx.pm
index 4543bfa1d9acba6e06c4dc3bbe3a91c309123404..d42d6fe76a273f5ed5f839a68a5388efc05eda1f 100644 (file)
--- a/lib/PublicInbox/OverIdx.pm
+++ b/lib/PublicInbox/OverIdx.pm
@@ -284,7 +284,7 @@ sub add_overview {
        my $dd = $smsg->to_doc_data;
        utf8::encode($dd);
        $dd = compress($dd);
-       add_over($self, [ @$smsg{qw(ts ds num)}, $mids, $refs, $xpath, $dd ]);
+       add_over($self, $smsg, $mids, $refs, $xpath, $dd);
 }
 
 sub _add_over {
@@ -311,10 +311,10 @@ sub _add_over {
 }
 
 sub add_over {
-       my ($self, $values) = @_;
-       my ($ts, $ds, $num, $mids, $refs, $xpath, $ddd) = @$values;
+       my ($self, $smsg, $mids, $refs, $xpath, $ddd) = @_;
        my $old_tid;
        my $vivified = 0;
+       my $num = $smsg->{num};
 
        begin_lazy($self);
        delete_by_num($self, $num, \$old_tid);
@@ -326,17 +326,17 @@ sub add_over {
                $v > 1 and warn "BUG: vivified multiple ($v) ghosts for $mid\n";
                $vivified += $v;
        }
-       my $tid = $vivified ? $old_tid : link_refs($self, $refs, $old_tid);
-       my $sid = sid($self, $xpath);
+       $smsg->{tid} = $vivified ? $old_tid : link_refs($self, $refs, $old_tid);
+       $smsg->{sid} = sid($self, $xpath);
        my $dbh = $self->{dbh};
        my $sth = $dbh->prepare_cached(<<'');
 INSERT INTO over (num, tid, sid, ts, ds, ddd)
 VALUES (?,?,?,?,?,?)
 
-       my $n = 0;
-       my @v = ($num, $tid, $sid, $ts, $ds);
-       foreach (@v) { $sth->bind_param(++$n, $_) }
-       $sth->bind_param(++$n, $ddd, SQL_BLOB);
+       my $nc = 1;
+       $sth->bind_param($nc, $num);
+       $sth->bind_param(++$nc, $smsg->{$_}) for (qw(tid sid ts ds));
+       $sth->bind_param(++$nc, $ddd, SQL_BLOB);
        $sth->execute;
        $sth = $dbh->prepare_cached(<<'');
 INSERT INTO id2num (id, num) VALUES (?,?)

diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm
index c18e19d4d4b45a3804c2d6f779ba5c8adcfe48d0..4cfb7b3846d5f6475478c1edb94893670720bd56 100644 (file)
--- a/lib/PublicInbox/Search.pm
+++ b/lib/PublicInbox/Search.pm
@@ -18,9 +18,9 @@ use constant {
        # added for public-inbox 1.6.0+
        BYTES => 3, # IMAP RFC822.SIZE
        UID => 4, # IMAP UID == NNTP article number == Xapian docid
+       THREADID => 5, # RFC 8474, RFC 8621
 
        # TODO
-       # THREADID => ?
        # REPLYCNT => ?, # IMAP ANSWERED
 
        # SCHEMA_VERSION history
@@ -47,7 +47,7 @@ use constant {
        #      public-inbox v1.5.0 adds (still SCHEMA_VERSION=15):
        #      * "lid:" and "l:" for List-Id searches
        #
-       #      v1.6.0 adds BYTES and UID values
+       #      v1.6.0 adds BYTES, UID and THREADID values
        SCHEMA_VERSION => 15,
 };
 

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 098fead78ccf0774cf0ea41c513242ce1c215d6d..baa6f41ad21b60b76e3b3ca56f81b9bb0efff53d 100644 (file)
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -356,6 +356,7 @@ sub add_xapian ($$$$) {
        add_val($doc, PublicInbox::Search::DT(), $dt);
        add_val($doc, PublicInbox::Search::BYTES(), $smsg->{bytes});
        add_val($doc, PublicInbox::Search::UID(), $smsg->{num});
+       add_val($doc, PublicInbox::Search::THREADID, $smsg->{tid});
 
        my $tg = term_generator($self);
        $tg->set_document($doc);

diff --git a/lib/PublicInbox/SearchIdxShard.pm b/lib/PublicInbox/SearchIdxShard.pm
index c0f8be8964939e5c95aa33b4894b0bde8cc05d40..f23d23d06f4772b2db98afc909fcb29792baeae1 100644 (file)
--- a/lib/PublicInbox/SearchIdxShard.pm
+++ b/lib/PublicInbox/SearchIdxShard.pm
@@ -68,8 +68,8 @@ sub shard_worker_loop ($$$$$) {
                } else {
                        chomp $line;
                        # n.b. $mid may contain spaces(!)
-                       my ($to_read, $bytes, $num, $blob, $ds, $ts, $mid) =
-                                                       split(/ /, $line, 7);
+                       my ($to_read, $bytes, $num, $blob, $ds, $ts, $tid, $mid)
+                               = split(/ /, $line, 8);
                        $self->begin_txn_lazy;
                        my $n = read($r, my $msg, $to_read) or die "read: $!\n";
                        $n == $to_read or die "short read: $n != $to_read\n";
@@ -79,6 +79,7 @@ sub shard_worker_loop ($$$$$) {
                                num => $num + 0,
                                blob => $blob,
                                mid => $mid,
+                               tid => $tid,
                                ds => $ds,
                                ts => $ts,
                        }, 'PublicInbox::Smsg';
@@ -93,7 +94,7 @@ sub index_raw {
        if (my $w = $self->{w}) {
                # mid must be last, it can contain spaces (but not LF)
                print $w join(' ', @$smsg{qw(raw_bytes bytes
-                                               num blob ds ts mid)}),
+                                               num blob ds ts tid mid)}),
                        "\n", $$msgref or die "failed to write shard $!\n";
        } else {
                if ($eml) {

diff --git a/lib/PublicInbox/Smsg.pm b/lib/PublicInbox/Smsg.pm
index 51226b8ed85428d51ccf29d46100ab1c48bb82b7..0a0384efaadf7811d372631885ec771345370c7b 100644 (file)
--- a/lib/PublicInbox/Smsg.pm
+++ b/lib/PublicInbox/Smsg.pm
@@ -82,7 +82,8 @@ sub psgi_cull ($) {
 
        # drop NNTP-only fields which aren't relevant to PSGI results:
        # saves ~80K on a 200 item search result:
-       delete @$self{qw(ts to cc bytes lines)};
+       # TODO: we may need to keep some of these for JMAP...
+       delete @$self{qw(ts tid to cc bytes lines)};
        $self;
 }
 

diff --git a/t/over.t b/t/over.t
index 734fdaa3604a55dd1da7df9ddb67ad48bd52c345..07672aa70a2e56c34ba3e83390f2899b7b3b6da8 100644 (file)
--- a/t/over.t
+++ b/t/over.t
@@ -40,22 +40,23 @@ $y = $over->create_ghost('NEVAR');
 is($y, $x + 1, 'integer tid for ghost increases');
 
 my $ddd = compress('');
+my $msg = sub { { ts => 0, ds => 0, num => $_[0] } };
 foreach my $s ('', undef) {
-       $over->add_over([0, 0, 98, [ 'a' ], [], $s, $ddd]);
-       $over->add_over([0, 0, 99, [ 'b' ], [], $s, $ddd]);
+       $over->add_over($msg->(98), [ 'a' ], [], $s, $ddd);
+       $over->add_over($msg->(99), [ 'b' ], [], $s, $ddd);
        my $msgs = [ map { $_->{num} } @{$over->get_thread('a')} ];
        is_deeply([98], $msgs,
                'messages not linked by empty subject');
 }
 
-$over->add_over([0, 0, 98, [ 'a' ], [], 's', $ddd]);
-$over->add_over([0, 0, 99, [ 'b' ], [], 's', $ddd]);
+$over->add_over($msg->(98), [ 'a' ], [], 's', $ddd);
+$over->add_over($msg->(99), [ 'b' ], [], 's', $ddd);
 foreach my $mid (qw(a b)) {
        my $msgs = [ map { $_->{num} } @{$over->get_thread('a')} ];
        is_deeply([98, 99], $msgs, 'linked messages by subject');
 }
-$over->add_over([0, 0, 98, [ 'a' ], [], 's', $ddd]);
-$over->add_over([0, 0, 99, [ 'b' ], ['a'], 'diff', $ddd]);
+$over->add_over($msg->(98), [ 'a' ], [], 's', $ddd);
+$over->add_over($msg->(99), [ 'b' ], ['a'], 'diff', $ddd);
 foreach my $mid (qw(a b)) {
        my $msgs = [ map { $_->{num} } @{$over->get_thread($mid)} ];
        is_deeply([98, 99], $msgs, "linked messages by Message-ID: <$mid>");