mbox: disable "&t" on existing Xapian until full reindex

[public-inbox.git] / lib / PublicInbox / SearchIdx.pm

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 5c39f3d620e3e774faab522c9de3113f995733d5..ade5575669c1b601162280369eb795aa3b0c40e1 100644 (file)
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -61,6 +61,10 @@ sub new {
        }, $class;
        $self->xpfx_init;
        $self->{-set_indexlevel_once} = 1 if $indexlevel eq 'medium';
+       if ($ibx->{-skip_docdata}) {
+               $self->{-set_skip_docdata_once} = 1;
+               $self->{-skip_docdata} = 1;
+       }
        $ibx->umask_prepare;
        if ($version == 1) {
                $self->{lock_path} = "$inboxdir/ssoma.lock";
@@ -127,6 +131,7 @@ sub idx_acquire {
                                ($is_shard && need_xapian($self)))) {
                        File::Path::mkpath($dir);
                        nodatacow_dir($dir);
+                       $self->{-set_has_threadid_once} = 1;
                }
        }
        return unless defined $flag;
@@ -352,6 +357,7 @@ sub add_xapian ($$$$) {
        add_val($doc, PublicInbox::Search::DT(), $dt);
        add_val($doc, PublicInbox::Search::BYTES(), $smsg->{bytes});
        add_val($doc, PublicInbox::Search::UID(), $smsg->{num});
+       add_val($doc, PublicInbox::Search::THREADID, $smsg->{tid});
 
        my $tg = term_generator($self);
        $tg->set_document($doc);
@@ -359,10 +365,18 @@ sub add_xapian ($$$$) {
 
        msg_iter($eml, \&index_xapian, [ $self, $doc ]);
        index_ids($self, $doc, $eml, $mids);
-       $smsg->{to} = $smsg->{cc} = ''; # WWW doesn't need these, only NNTP
-       PublicInbox::OverIdx::parse_references($smsg, $eml, $mids);
-       my $data = $smsg->to_doc_data;
-       $doc->set_data($data);
+
+       # by default, we maintain compatibility with v1.5.0 and earlier
+       # by writing to docdata.glass, users who never exect to downgrade can
+       # use --skip-docdata
+       if (!$self->{-skip_docdata}) {
+               # WWW doesn't need {to} or {cc}, only NNTP
+               $smsg->{to} = $smsg->{cc} = '';
+               PublicInbox::OverIdx::parse_references($smsg, $eml, $mids);
+               my $data = $smsg->to_doc_data;
+               $doc->set_data($data);
+       }
+
        if (my $altid = $self->{-altid}) {
                foreach my $alt (@$altid) {
                        my $pfx = $alt->{xprefix};
@@ -577,9 +591,17 @@ sub v1_checkpoint ($$;$) {
 
        $self->{mm}->{dbh}->commit;
        if ($newest && need_xapian($self)) {
-               my $cur = $self->{xdb}->get_metadata('last_commit');
+               my $xdb = $self->{xdb};
+               my $cur = $xdb->get_metadata('last_commit');
                if (need_update($self, $cur, $newest)) {
-                       $self->{xdb}->set_metadata('last_commit', $newest);
+                       $xdb->set_metadata('last_commit', $newest);
+               }
+
+               # let SearchView know a full --reindex was done so it can
+               # generate ->has_threadid-dependent links
+               if ($sync->{reindex} && !ref($sync->{reindex})) {
+                       my $n = $xdb->get_metadata('has_threadid');
+                       $xdb->set_metadata('has_threadid', '1') if $n ne '1';
                }
        }
 
@@ -781,40 +803,6 @@ sub DESTROY {
        $_[0]->{lockfh} = undef;
 }
 
-# remote_* subs are only used by SearchIdxPart
-sub remote_commit {
-       my ($self) = @_;
-       if (my $w = $self->{w}) {
-               print $w "commit\n" or die "failed to write commit: $!";
-       } else {
-               $self->commit_txn_lazy;
-       }
-}
-
-sub remote_close {
-       my ($self) = @_;
-       if (my $w = delete $self->{w}) {
-               my $pid = delete $self->{pid} or die "no process to wait on\n";
-               print $w "close\n" or die "failed to write to pid:$pid: $!\n";
-               close $w or die "failed to close pipe for pid:$pid: $!\n";
-               waitpid($pid, 0) == $pid or die "remote process did not finish";
-               $? == 0 or die ref($self)." pid:$pid exited with: $?";
-       } else {
-               die "transaction in progress $self\n" if $self->{txn};
-               idx_release($self) if $self->{xdb};
-       }
-}
-
-sub remote_remove {
-       my ($self, $oid, $num) = @_;
-       if (my $w = $self->{w}) {
-               # triggers remove_by_oid in a shard
-               print $w "D $oid $num\n" or die "failed to write remove $!";
-       } else {
-               $self->remove_by_oid($oid, $num);
-       }
-}
-
 sub _begin_txn {
        my ($self) = @_;
        my $xdb = $self->{xdb} || idx_acquire($self);
@@ -831,23 +819,31 @@ sub begin_txn_lazy {
 
 # store 'indexlevel=medium' in v2 shard=0 and v1 (only one shard)
 # This metadata is read by Admin::detect_indexlevel:
-sub set_indexlevel {
+sub set_metadata_once {
        my ($self) = @_;
 
-       if (!$self->{shard} && # undef or 0, not >0
-                       delete($self->{-set_indexlevel_once})) {
-               my $xdb = $self->{xdb};
+       return if $self->{shard}; # only continue if undef or 0, not >0
+       my $xdb = $self->{xdb};
+
+       if (delete($self->{-set_has_threadid_once})) {
+               $xdb->set_metadata('has_threadid', '1');
+       }
+       if (delete($self->{-set_indexlevel_once})) {
                my $level = $xdb->get_metadata('indexlevel');
                if (!$level || $level ne 'medium') {
                        $xdb->set_metadata('indexlevel', 'medium');
                }
        }
+       if (delete($self->{-set_skip_docdata_once})) {
+               $xdb->get_metadata('skip_docdata') or
+                       $xdb->set_metadata('skip_docdata', '1');
+       }
 }
 
 sub _commit_txn {
        my ($self) = @_;
        if (my $xdb = $self->{xdb}) {
-               set_indexlevel($self);
+               set_metadata_once($self);
                $xdb->commit_transaction;
        }
        $self->{over}->commit_lazy if $self->{over};