From 826e0992d956aa60711e779afeb8493884c41cc2 Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Fri, 7 Aug 2020 10:52:14 +0000 Subject: [PATCH] v2writable: fix batch size accounting We need to account for whether shard parallelization is enabled or not, since users of parallelization are expected to have more RAM. --- lib/PublicInbox/V2Writable.pm | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm index a029fe4c..f7a318e5 100644 --- a/lib/PublicInbox/V2Writable.pm +++ b/lib/PublicInbox/V2Writable.pm @@ -152,6 +152,12 @@ sub add { $self->{ibx}->with_umask(\&_add, $self, $eml, $check_cb); } +sub batch_bytes ($) { + my ($self) = @_; + ($self->{parallel} ? $self->{shards} : 1) * + $PublicInbox::SearchIdx::BATCH_BYTES; +} + # indexes a message, returns true if checkpointing is needed sub do_idx ($$$$) { my ($self, $msgref, $mime, $smsg) = @_; @@ -160,7 +166,7 @@ sub do_idx ($$$$) { my $idx = idx_shard($self, $smsg->{num} % $self->{shards}); $idx->index_raw($msgref, $mime, $smsg); my $n = $self->{transact_bytes} += $smsg->{raw_bytes}; - $n >= ($PublicInbox::SearchIdx::BATCH_BYTES * $self->{shards}); + $n >= batch_bytes($self); } sub _add { @@ -1195,7 +1201,7 @@ sub index_xap_step ($$$;$) { my $ibx = $self->{ibx}; my $all = $ibx->git; my $over = $ibx->over; - my $batch_bytes = $PublicInbox::SearchIdx::BATCH_BYTES; + my $batch_bytes = batch_bytes($self); $step //= $self->{shards}; my $end = $sync->{art_end}; if (my $pr = $sync->{-opt}->{-progress}) { -- 2.44.0