v2writable: implement remove correctly

[public-inbox.git] / lib / PublicInbox / SearchIdx.pm

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 71469a9566e774c1c56e990e63e868bff110ed44..ae2544da18a32e5472077caee939080ca639637b 100644 (file)
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -406,29 +406,63 @@ sub add_message {
        $doc_id;
 }
 
-# returns deleted doc_id on success, undef on missing
+sub batch_do {
+       my ($self, $termval, $cb) = @_;
+       my $batch_size = 1000; # don't let @ids grow too large to avoid OOM
+       while (1) {
+               my ($head, $tail) = $self->find_doc_ids($termval);
+               return if $head == $tail;
+               my @ids;
+               for (; $head != $tail && @ids < $batch_size; $head->inc) {
+                       push @ids, $head->get_docid;
+               }
+               $cb->(\@ids);
+       }
+}
+
 sub remove_message {
        my ($self, $mid) = @_;
        my $db = $self->{xdb};
-       my $doc_id;
+       my $called;
        $mid = mid_clean($mid);
 
        eval {
-               my ($head, $tail) = $self->find_doc_ids('Q' . $mid);
-               if ($head->equal($tail)) {
-                       warn "cannot remove non-existent <$mid>\n";
-               }
-               for (; $head != $tail; $head->inc) {
-                       my $docid = $head->get_docid;
-                       $db->delete_document($docid);
-               }
+               batch_do($self, 'Q' . $mid, sub {
+                       my ($ids) = @_;
+                       $db->delete_document($_) for @$ids;
+                       $called = 1;
+               });
        };
-
        if ($@) {
                warn "failed to remove message <$mid>: $@\n";
-               return undef;
+       } elsif (!$called) {
+               warn "cannot remove non-existent <$mid>\n";
        }
-       $doc_id;
+}
+
+# MID is a hint in V2
+sub remove_by_oid {
+       my ($self, $oid, $mid) = @_;
+       my $db = $self->{xdb};
+
+       # XXX careful, we cannot use batch_do here since we conditionally
+       # delete documents based on other factors, so we cannot call
+       # find_doc_ids twice.
+       my ($head, $tail) = $self->find_doc_ids('Q' . $mid);
+       return if $head == $tail;
+
+       # there is only ONE element in @delete unless we
+       # have bugs in our v2writable deduplication check
+       my @delete;
+       for (; $head != $tail; $head->inc) {
+               my $docid = $head->get_docid;
+               my $doc = $db->get_document($docid);
+               my $smsg = PublicInbox::SearchMsg->wrap($doc, $mid);
+               $smsg->load_expand;
+               push(@delete, $docid) if $smsg->{blob} eq $oid;
+       }
+       $db->delete_document($_) foreach @delete;
+       scalar(@delete);
 }
 
 sub term_generator { # write-only
@@ -795,29 +829,22 @@ sub merge_threads {
        my ($self, $winner_tid, $loser_tid) = @_;
        return if $winner_tid == $loser_tid;
        my $db = $self->{xdb};
-
-       my $batch_size = 1000; # don't let @ids grow too large to avoid OOM
-       while (1) {
-               my ($head, $tail) = $self->find_doc_ids('G' . $loser_tid);
-               return if $head == $tail;
-               my @ids;
-               for (; $head != $tail && @ids < $batch_size; $head->inc) {
-                       push @ids, $head->get_docid;
-               }
-               foreach my $docid (@ids) {
+       batch_do($self, 'G' . $loser_tid, sub {
+               my ($ids) = @_;
+               foreach my $docid (@$ids) {
                        my $doc = $db->get_document($docid);
                        $doc->remove_term('G' . $loser_tid);
                        $doc->add_boolean_term('G' . $winner_tid);
                        $db->replace_document($docid, $doc);
                }
-       }
+       });
 }
 
 sub _read_git_config_perm {
        my ($self) = @_;
        my @cmd = qw(config);
        if ($self->{version} == 2) {
-               push @cmd, "--file=$self->{mainrepo}/inbox-config";
+               push @cmd, "--file=$self->{mainrepo}/all.git/config";
        }
        my $fh = $self->{git}->popen(@cmd, 'core.sharedRepository');
        local $/ = "\n";
@@ -894,4 +921,11 @@ sub remote_close {
        $? == 0 or die ref($self)." pid:$pid exited with: $?";
 }
 
+# triggers remove_by_oid in partition or skeleton
+sub remote_remove {
+       my ($self, $oid, $mid) = @_;
+       print { $self->{w} } "D $oid $mid\n" or
+                       die "failed to write remove $!";
+}
+
 1;