No ext_urls

[public-inbox.git] / lib / PublicInbox / LeiStore.pm

diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index f1316229bb3205c9395c345de49d4f89f003561b..fce15a724b511d15c49a3a2c196eb4d9e68291d7 100644 (file)
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 #
 # Local storage (cache/memo) for lei(1), suitable for personal/private
@@ -255,21 +255,16 @@ sub remove_eml_vmd { # remove just the VMD
 
 sub _lms_rw ($) { # it is important to have eidx processes open before lms
        my ($self) = @_;
-       my ($eidx, $tl) = eidx_init($self);
-       $self->{lms} //= do {
+       $self->{lms} // do {
                require PublicInbox::LeiMailSync;
+               my ($eidx, $tl) = eidx_init($self);
                my $f = "$self->{priv_eidx}->{topdir}/mail_sync.sqlite3";
                my $lms = PublicInbox::LeiMailSync->new($f);
                $lms->lms_write_prepare;
-               $lms;
+               $self->{lms} = $lms;
        };
 }
 
-sub set_sync_info {
-       my ($self, $oidhex, $folder, $id) = @_;
-       _lms_rw($self)->set_src(pack('H*', $oidhex), $folder, $id);
-}
-
 sub _remove_if_local { # git->cat_async arg
        my ($bref, $oidhex, $type, $size, $self) = @_;
        $self->{im}->remove($bref) if $bref;
@@ -340,6 +335,46 @@ sub _docids_and_maybe_kw ($$) {
        ($docids, [ sort keys %$kw ]);
 }
 
+sub _reindex_1 { # git->cat_async callback
+       my ($bref, $hex, $type, $size, $smsg) = @_;
+       my $self = delete $smsg->{-sto};
+       my ($eidx, $tl) = eidx_init($self);
+       $bref //= _lms_rw($self)->local_blob($hex, 1);
+       if ($bref) {
+               my $eml = PublicInbox::Eml->new($bref);
+               $smsg->{-merge_vmd} = 1; # preserve existing keywords
+               $eidx->idx_shard($smsg->{num})->index_eml($eml, $smsg);
+       } elsif ($type eq 'missing') {
+               # pre-release/buggy lei may've indexed external-only msgs,
+               # try to correct that, here
+               warn("E: missing $hex, culling (ancient lei artifact?)\n");
+               $smsg->{to} = $smsg->{cc} = $smsg->{from} = '';
+               $smsg->{bytes} = 0;
+               $eidx->{oidx}->update_blob($smsg, '');
+               my $eml = PublicInbox::Eml->new("\r\n\r\n");
+               $eidx->idx_shard($smsg->{num})->index_eml($eml, $smsg);
+       } else {
+               warn("E: $type $hex\n");
+       }
+}
+
+sub reindex_art {
+       my ($self, $art) = @_;
+       my ($eidx, $tl) = eidx_init($self);
+       my $smsg = $eidx->{oidx}->get_art($art) // return;
+       return if $smsg->{bytes} == 0; # external-only message
+       $smsg->{-sto} = $self;
+       $eidx->git->cat_async($smsg->{blob} // die("no blob (#$art)"),
+                               \&_reindex_1, $smsg);
+}
+
+sub reindex_done {
+       my ($self) = @_;
+       my ($eidx, $tl) = eidx_init($self);
+       $eidx->git->async_wait_all;
+       # ->done to be called via sto_done_request
+}
+
 sub add_eml {
        my ($self, $eml, $vmd, $xoids) = @_;
        my $im = $self->{-fake_im} // $self->importer; # may create new epoch
@@ -349,7 +384,7 @@ sub add_eml {
        $smsg->{-eidx_git} = $eidx->git if !$self->{-fake_im};
        my $im_mark = $im->add($eml, undef, $smsg);
        if ($vmd && $vmd->{sync_info}) {
-               set_sync_info($self, $smsg->{blob}, @{$vmd->{sync_info}});
+               _lms_rw($self)->set_src($smsg->oidbin, @{$vmd->{sync_info}});
        }
        unless ($im_mark) { # duplicate blob returns undef
                return unless wantarray;
@@ -569,9 +604,8 @@ sub recv_and_run {
        $self->SUPER::recv_and_run(@args);
 }
 
-sub _sto_atexit { # dwaitpid callback
-       my ($args, $pid) = @_;
-       my $self = $args->[0];
+sub _sto_atexit { # awaitpid cb
+       my ($pid) = @_;
        warn "lei/store PID:$pid died \$?=$?\n" if $?;
 }
 
@@ -590,8 +624,7 @@ sub write_prepare {
                                        lei => $lei,
                                        -err_wr => $w,
                                        to_close => [ $r ],
-                               });
-               $self->wq_wait_async(\&_sto_atexit); # outlives $lei
+                               }, \&_sto_atexit);
                require PublicInbox::LeiStoreErr;
                PublicInbox::LeiStoreErr->new($r, $lei);
        }