lei rm: new command to remove messages from index

[public-inbox.git] / lib / PublicInbox / LeiStore.pm

diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index fcc9224d4644d7c2c6d97a96a99128155539c329..6888afb40d3568a9003e9ccf51d4e5d171d5aeaa 100644 (file)
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -88,6 +88,7 @@ sub importer {
                $self->checkpoint;
                $max = $self->git_epoch_max + 1;
        }
+       my (undef, $tl) = eidx_init($self); # acquire lock
        my $pfx = $self->git_pfx;
        $max //= $self->git_epoch_max;
        while (1) {
@@ -97,7 +98,9 @@ sub importer {
                my $git = PublicInbox::Git->new($latest);
                if (!$old) {
                        $git->qx(qw(config core.sharedRepository 0600));
-                       $self->done; # force eidx_init on next round
+                       $self->done; # unlock
+                       # re-acquire lock, update alternates for new epoch
+                       (undef, $tl) = eidx_init($self);
                }
                my $packed_bytes = $git->packed_bytes;
                my $unpacked_bytes = $packed_bytes / $self->packing_factor;
@@ -130,7 +133,7 @@ sub eidx_init {
        my $tl = wantarray && $self->{-err_wr} ?
                        PublicInbox::OnDestroy->new($$, \&_tail_err, $self) :
                        undef;
-       $eidx->idx_init({-private => 1});
+       $eidx->idx_init({-private => 1}); # acquires lock
        wantarray ? ($eidx, $tl) : $eidx;
 }
 
@@ -180,7 +183,7 @@ sub add_eml_vmd {
        \@docids;
 }
 
-sub remove_eml_vmd {
+sub remove_eml_vmd { # remove just the VMD
        my ($self, $eml, $vmd) = @_;
        my ($eidx, $tl) = eidx_init($self);
        my @docids = _docids_for($self, $eml);
@@ -190,26 +193,54 @@ sub remove_eml_vmd {
        \@docids;
 }
 
-sub set_sync_info ($$$) {
-       my ($self, $oidhex, $sync_info) = @_;
+sub set_sync_info {
+       my ($self, $oidhex, $folder, $id) = @_;
        ($self->{lms} //= do {
                require PublicInbox::LeiMailSync;
                my $f = "$self->{priv_eidx}->{topdir}/mail_sync.sqlite3";
                my $lms = PublicInbox::LeiMailSync->new($f);
                $lms->lms_begin;
                $lms;
-       })->set_src($oidhex, @$sync_info);
+       })->set_src($oidhex, $folder, $id);
+}
+
+sub _remove_if_local { # git->cat_async arg
+       my ($bref, $oidhex, $type, $size, $self) = @_;
+       $self->{im}->remove($bref) if $bref;
+}
+
+# remove the entire message from the index, does not touch mail_sync.sqlite3
+sub remove_eml {
+       my ($self, $eml) = @_;
+       my $im = $self->importer; # may create new epoch
+       my ($eidx, $tl) = eidx_init($self);
+       my $oidx = $eidx->{oidx};
+       my @docids = _docids_for($self, $eml);
+       my $git = $eidx->git;
+       for my $docid (@docids) {
+               my $xr3 = $oidx->get_xref3($docid, 1);
+               for my $row (@$xr3) {
+                       my (undef, undef, $oidbin) = @$row;
+                       my $oidhex = unpack('H*', $oidbin);
+                       $git->cat_async($oidhex, \&_remove_if_local, $self);
+               }
+               $eidx->idx_shard($docid)->ipc_do('xdb_remove', $docid);
+               $oidx->delete_by_num($docid);
+       }
+       $git->cat_async_wait;
+       \@docids;
 }
 
 sub add_eml {
        my ($self, $eml, $vmd, $xoids) = @_;
-       my $im = $self->importer; # may create new epoch
-       my ($eidx, $tl) = eidx_init($self); # updates/writes alternates file
+       my $im = $self->{-fake_im} // $self->importer; # may create new epoch
+       my ($eidx, $tl) = eidx_init($self);
        my $oidx = $eidx->{oidx}; # PublicInbox::Import::add checks this
        my $smsg = bless { -oidx => $oidx }, 'PublicInbox::Smsg';
+       $smsg->{-eidx_git} = $eidx->git if !$self->{-fake_im};
        my $im_mark = $im->add($eml, undef, $smsg);
        if ($vmd && $vmd->{sync_info}) {
-               set_sync_info($self, $smsg->{blob}, $vmd->{sync_info});
+               set_sync_info($self, $smsg->{blob}, @{$vmd->{sync_info}});
        }
        $im_mark or return; # duplicate blob returns undef
 
@@ -273,6 +304,13 @@ sub set_eml {
                set_eml_vmd($self, $eml, $vmd);
 }
 
+sub index_eml_only {
+       my ($self, $eml, $vmd, $xoids) = @_;
+       require PublicInbox::FakeImport;
+       local $self->{-fake_im} = PublicInbox::FakeImport->new;
+       set_eml($self, $eml, $vmd, $xoids);
+}
+
 sub _external_only ($$$) {
        my ($self, $xoids, $eml) = @_;
        my $eidx = $self->{priv_eidx};
@@ -420,20 +458,15 @@ sub write_prepare {
                my $d = $lei->store_path;
                $self->ipc_lock_init("$d/ipc.lock");
                substr($d, -length('/lei/store'), 10, '');
-               my $err_pipe;
-               unless ($lei->{oneshot}) {
-                       pipe(my ($r, $w)) or die "pipe: $!";
-                       $err_pipe = [ $r, $w ];
-               }
+               pipe(my ($r, $w)) or die "pipe: $!";
+               my $err_pipe = [ $r, $w ];
                # Mail we import into lei are private, so headers filtered out
                # by -mda for public mail are not appropriate
                local @PublicInbox::MDA::BAD_HEADERS = ();
                $self->ipc_worker_spawn("lei/store $d", $lei->oldset,
                                        { lei => $lei, err_pipe => $err_pipe });
-               if ($err_pipe) {
-                       require PublicInbox::LeiStoreErr;
-                       PublicInbox::LeiStoreErr->new($err_pipe->[0], $lei);
-               }
+               require PublicInbox::LeiStoreErr;
+               PublicInbox::LeiStoreErr->new($err_pipe->[0], $lei);
        }
        $lei->{sto} = $self;
 }