searchidx: put all shard-related stuff in SearchIdxShard.pm

[public-inbox.git] / lib / PublicInbox / SearchIdx.pm

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 7f2447fe5e4ab970c83f633f6fca8969cc2691c0..098fead78ccf0774cf0ea41c513242ce1c215d6d 100644 (file)
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -61,6 +61,10 @@ sub new {
        }, $class;
        $self->xpfx_init;
        $self->{-set_indexlevel_once} = 1 if $indexlevel eq 'medium';
+       if ($ibx->{-skip_docdata}) {
+               $self->{-set_skip_docdata_once} = 1;
+               $self->{-skip_docdata} = 1;
+       }
        $ibx->umask_prepare;
        if ($version == 1) {
                $self->{lock_path} = "$inboxdir/ssoma.lock";
@@ -359,10 +363,18 @@ sub add_xapian ($$$$) {
 
        msg_iter($eml, \&index_xapian, [ $self, $doc ]);
        index_ids($self, $doc, $eml, $mids);
-       $smsg->{to} = $smsg->{cc} = ''; # WWW doesn't need these, only NNTP
-       PublicInbox::OverIdx::parse_references($smsg, $eml, $mids);
-       my $data = $smsg->to_doc_data;
-       $doc->set_data($data);
+
+       # by default, we maintain compatibility with v1.5.0 and earlier
+       # by writing to docdata.glass, users who never exect to downgrade can
+       # use --skip-docdata
+       if (!$self->{-skip_docdata}) {
+               # WWW doesn't need {to} or {cc}, only NNTP
+               $smsg->{to} = $smsg->{cc} = '';
+               PublicInbox::OverIdx::parse_references($smsg, $eml, $mids);
+               my $data = $smsg->to_doc_data;
+               $doc->set_data($data);
+       }
+
        if (my $altid = $self->{-altid}) {
                foreach my $alt (@$altid) {
                        my $pfx = $alt->{xprefix};
@@ -539,11 +551,11 @@ sub unindex_both { # git->cat_async callback
 
 # called by public-inbox-index
 sub index_sync {
-       my ($self, $opts) = @_;
-       delete $self->{lock_path} if $opts->{-skip_lock};
-       $self->{ibx}->with_umask(\&_index_sync, $self, $opts);
-       if ($opts->{reindex}) {
-               my %again = %$opts;
+       my ($self, $opt) = @_;
+       delete $self->{lock_path} if $opt->{-skip_lock};
+       $self->{ibx}->with_umask(\&_index_sync, $self, $opt);
+       if ($opt->{reindex}) {
+               my %again = %$opt;
                delete @again{qw(rethread reindex)};
                index_sync($self, \%again);
        }
@@ -745,15 +757,15 @@ sub reindex_from ($$) {
 
 # indexes all unindexed messages (v1 only)
 sub _index_sync {
-       my ($self, $opts) = @_;
-       my $tip = $opts->{ref} || 'HEAD';
+       my ($self, $opt) = @_;
+       my $tip = $opt->{ref} || 'HEAD';
        my $git = $self->{ibx}->git;
-       $self->{batch_bytes} = $opts->{batch_size} // $BATCH_BYTES;
+       $self->{batch_bytes} = $opt->{batch_size} // $BATCH_BYTES;
        $git->batch_prepare;
-       my $pr = $opts->{-progress};
-       my $sync = { reindex => $opts->{reindex}, -opt => $opts };
+       my $pr = $opt->{-progress};
+       my $sync = { reindex => $opt->{reindex}, -opt => $opt };
        my $xdb = $self->begin_txn_lazy;
-       $self->{over}->rethread_prepare($opts);
+       $self->{over}->rethread_prepare($opt);
        my $mm = _msgmap_init($self);
        if ($sync->{reindex}) {
                my $last = $mm->last_commit;
@@ -781,40 +793,6 @@ sub DESTROY {
        $_[0]->{lockfh} = undef;
 }
 
-# remote_* subs are only used by SearchIdxPart
-sub remote_commit {
-       my ($self) = @_;
-       if (my $w = $self->{w}) {
-               print $w "commit\n" or die "failed to write commit: $!";
-       } else {
-               $self->commit_txn_lazy;
-       }
-}
-
-sub remote_close {
-       my ($self) = @_;
-       if (my $w = delete $self->{w}) {
-               my $pid = delete $self->{pid} or die "no process to wait on\n";
-               print $w "close\n" or die "failed to write to pid:$pid: $!\n";
-               close $w or die "failed to close pipe for pid:$pid: $!\n";
-               waitpid($pid, 0) == $pid or die "remote process did not finish";
-               $? == 0 or die ref($self)." pid:$pid exited with: $?";
-       } else {
-               die "transaction in progress $self\n" if $self->{txn};
-               idx_release($self) if $self->{xdb};
-       }
-}
-
-sub remote_remove {
-       my ($self, $oid, $num) = @_;
-       if (my $w = $self->{w}) {
-               # triggers remove_by_oid in a shard
-               print $w "D $oid $num\n" or die "failed to write remove $!";
-       } else {
-               $self->remove_by_oid($oid, $num);
-       }
-}
-
 sub _begin_txn {
        my ($self) = @_;
        my $xdb = $self->{xdb} || idx_acquire($self);
@@ -831,23 +809,28 @@ sub begin_txn_lazy {
 
 # store 'indexlevel=medium' in v2 shard=0 and v1 (only one shard)
 # This metadata is read by Admin::detect_indexlevel:
-sub set_indexlevel {
+sub set_metadata_once {
        my ($self) = @_;
 
-       if (!$self->{shard} && # undef or 0, not >0
-                       delete($self->{-set_indexlevel_once})) {
-               my $xdb = $self->{xdb};
+       return if $self->{shard}; # only continue if undef or 0, not >0
+       my $xdb = $self->{xdb};
+
+       if (delete($self->{-set_indexlevel_once})) {
                my $level = $xdb->get_metadata('indexlevel');
                if (!$level || $level ne 'medium') {
                        $xdb->set_metadata('indexlevel', 'medium');
                }
        }
+       if (delete($self->{-set_skip_docdata_once})) {
+               $xdb->get_metadata('skip_docdata') or
+                       $xdb->set_metadata('skip_docdata', '1');
+       }
 }
 
 sub _commit_txn {
        my ($self) = @_;
        if (my $xdb = $self->{xdb}) {
-               set_indexlevel($self);
+               set_metadata_once($self);
                $xdb->commit_transaction;
        }
        $self->{over}->commit_lazy if $self->{over};