v2writable: support purging messages from git entirely

[public-inbox.git] / lib / PublicInbox / V2Writable.pm

diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index b04e6fbfe892dc1227b43c119f93713f38008a86..ef9867de35ab225da985cdef5d3052832fcfd422 100644 (file)
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -173,7 +173,7 @@ sub num_for_harder {
                        $num = $self->{skel}->{mm}->mid_insert($$mid0);
                }
        }
-       PublicInbox::Import::prepend_mid($hdr, $$mid0);
+       PublicInbox::Import::append_mid($hdr, $$mid0);
        $num;
 }
 
@@ -209,11 +209,22 @@ sub idx_init {
        $skel->_msgmap_init->{dbh}->begin_work;
 }
 
-sub remove {
-       my ($self, $mime, $cmt_msg) = @_;
+sub purge_oids {
+       my ($self, $purge) = @_; # $purge = { $object_id => 1, ... }
+       $self->done;
+       my $pfx = "$self->{-inbox}->{mainrepo}/git";
+       foreach my $i (0..$self->{max_git}) {
+               my $git = PublicInbox::Git->new("$pfx/$i.git");
+               my $im = $self->import_init($git, 0);
+               $im->purge_oids($purge);
+       }
+}
+
+sub remove_internal {
+       my ($self, $mime, $cmt_msg, $purge) = @_;
        $self->barrier;
        $self->idx_init;
-       my $im = $self->importer;
+       my $im = $self->importer unless $purge;
        my $ibx = $self->{-inbox};
        my $srch = $ibx->search;
        my $cid = content_id($mime);
@@ -245,11 +256,15 @@ sub remove {
                                # no bugs in our deduplication code:
                                $removed = $smsg;
                                $removed->{mime} = $cur;
-                               $im->remove(\$orig, $cmt_msg);
+                               my $oid = $smsg->{blob};
+                               if ($purge) {
+                                       $purge->{$oid} = 1;
+                               } else {
+                                       $im->remove(\$orig, $cmt_msg);
+                               }
                                $orig = undef;
                                $removed->num; # memoize this for callers
 
-                               my $oid = $smsg->{blob};
                                foreach my $idx (@$parts, $skel) {
                                        $idx->remote_remove($oid, $mid);
                                }
@@ -258,9 +273,23 @@ sub remove {
                });
                $self->barrier;
        }
+       if ($purge && scalar keys %$purge) {
+               purge_oids($self, $purge);
+       }
        $removed;
 }
 
+sub remove {
+       my ($self, $mime, $cmt_msg) = @_;
+       remove_internal($self, $mime, $cmt_msg);
+}
+
+sub purge {
+       my ($self, $mime) = @_;
+       remove_internal($self, $mime, undef, {});
+}
+
+
 sub done {
        my ($self) = @_;
        my $locked = defined $self->{idx_parts};
@@ -495,7 +524,7 @@ sub mark_deleted {
        my $mids = mids($mime->header_obj);
        my $cid = content_id($mime);
        foreach my $mid (@$mids) {
-               $D->{$mid.$cid} = 1;
+               $D->{"$mid\0$cid"} = 1;
        }
 }
 
@@ -513,7 +542,7 @@ sub reindex_oid {
        my $num = -1;
        my $del = 0;
        foreach my $mid (@$mids) {
-               $del += (delete $D->{$mid.$cid} || 0);
+               $del += (delete $D->{"$mid\0$cid"} || 0);
                my $n = $mm_tmp->num_for($mid);
                if (defined $n && $n > $num) {
                        $mid0 = $mid;
@@ -633,6 +662,14 @@ sub reindex {
        }
        my ($min, $max) = $mm_tmp->minmax;
        defined $max and die "leftover article numbers at $min..$max\n";
+       my @d = sort keys %$D;
+       if (@d) {
+               warn "BUG: ", scalar(@d)," unseen deleted messages marked\n";
+               foreach (@d) {
+                       my ($mid, undef) = split(/\0/, $_, 2);
+                       warn "<$mid>\n";
+               }
+       }
 }
 
 1;