searchidx: do not modify Xapian DB while iterating

[public-inbox.git] / lib / PublicInbox / SearchIdx.pm

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 3259413789a1b4e3da5099fdf0e6a5146f209e5d..f4238fe8d5380aa1c2eecaaab0d01209b3418650 100644 (file)
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -740,15 +740,22 @@ sub create_ghost {
 sub merge_threads {
        my ($self, $winner_tid, $loser_tid) = @_;
        return if $winner_tid == $loser_tid;
-       my ($head, $tail) = $self->find_doc_ids('G' . $loser_tid);
        my $db = $self->{xdb};
 
-       for (; $head != $tail; $head->inc) {
-               my $docid = $head->get_docid;
-               my $doc = $db->get_document($docid);
-               $doc->remove_term('G' . $loser_tid);
-               $doc->add_term('G' . $winner_tid);
-               $db->replace_document($docid, $doc);
+       my $batch_size = 1000; # don't let @ids grow too large to avoid OOM
+       while (1) {
+               my ($head, $tail) = $self->find_doc_ids('G' . $loser_tid);
+               return if $head == $tail;
+               my @ids;
+               for (; $head != $tail && @ids < $batch_size; $head->inc) {
+                       push @ids, $head->get_docid;
+               }
+               foreach my $docid (@ids) {
+                       my $doc = $db->get_document($docid);
+                       $doc->remove_term('G' . $loser_tid);
+                       $doc->add_term('G' . $winner_tid);
+                       $db->replace_document($docid, $doc);
+               }
        }
 }