index: support --max-size / publicinbox.indexMaxSize

[public-inbox.git] / lib / PublicInbox / V2Writable.pm

diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index b45d272263b33b72062a054cc23dea7593c558fd..01b8bed6d7aabbc9a96e220a0895cfe2c553c1d5 100644 (file)
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -20,6 +20,7 @@ use PublicInbox::Msgmap;
 use PublicInbox::Spawn qw(spawn popen_rd);
 use PublicInbox::SearchIdx;
 use PublicInbox::MsgTime qw(msg_timestamp msg_datestamp);
+use PublicInbox::MultiMidQueue;
 use IO::Handle; # ->autoflush
 use File::Temp qw(tempfile);
 
@@ -119,6 +120,7 @@ sub new {
                last_commit => [], # git repo -> commit
        };
        $self->{shards} = count_shards($self) || nproc_shards($creat);
+       $self->{index_max_size} = $v2ibx->{index_max_size};
        bless $self, $class;
 }
 
@@ -729,9 +731,8 @@ sub fill_alternates ($$) {
 sub git_init {
        my ($self, $epoch) = @_;
        my $git_dir = "$self->{-inbox}->{inboxdir}/git/$epoch.git";
-       my @cmd = (qw(git init --bare -q), $git_dir);
-       PublicInbox::Import::run_die(\@cmd);
-       @cmd = (qw/git config/, "--file=$git_dir/config",
+       PublicInbox::Import::init_bare($git_dir);
+       my @cmd = (qw/git config/, "--file=$git_dir/config",
                        'include.path', '../../all.git/config');
        PublicInbox::Import::run_die(\@cmd);
        fill_alternates($self, $epoch);
@@ -867,6 +868,7 @@ sub atfork_child {
 
 sub mark_deleted ($$$$) {
        my ($self, $sync, $git, $oid) = @_;
+       return if PublicInbox::SearchIdx::too_big($self, $git, $oid);
        my $msgref = $git->cat_file($oid);
        my $mime = PublicInbox::MIME->new($$msgref);
        my $mids = mids($mime->header_obj);
@@ -979,32 +981,21 @@ sub check_unindexed ($$$) {
        }
 }
 
-# reuse Msgmap to store num => oid mapping (rather than num => mid)
-sub multi_mid_q_new () {
-       my ($fh, $fn) = tempfile('multi_mid-XXXXXXX', EXLOCK => 0, TMPDIR => 1);
-       my $multi_mid = PublicInbox::Msgmap->new_file($fn, 1);
-       $multi_mid->{dbh}->do('PRAGMA synchronous = OFF');
-       # for Msgmap->DESTROY:
-       $multi_mid->{tmp_name} = $fn;
-       $multi_mid->{pid} = $$;
-       close $fh or die "failed to close $fn: $!";
-       $multi_mid
-}
-
-sub multi_mid_q_push ($$) {
-       my ($sync, $oid) = @_;
-       my $multi_mid = $sync->{multi_mid} //= multi_mid_q_new();
+sub multi_mid_q_push ($$$) {
+       my ($self, $sync, $oid) = @_;
+       my $multi_mid = $sync->{multi_mid} //= PublicInbox::MultiMidQueue->new;
        if ($sync->{reindex}) { # no regen on reindex
-               $multi_mid->mid_insert($oid);
+               $multi_mid->push_oid($oid, $self);
        } else {
                my $num = $sync->{regen}--;
                die "BUG: ran out of article numbers" if $num <= 0;
-               $multi_mid->mid_set($num, $oid);
+               $multi_mid->set_oid($num, $oid, $self);
        }
 }
 
 sub reindex_oid ($$$$) {
        my ($self, $sync, $git, $oid) = @_;
+       return if PublicInbox::SearchIdx::too_big($self, $git, $oid);
        my ($num, $mid0, $len);
        my $msgref = $git->cat_file($oid, \$len);
        return if $len == 0; # purged
@@ -1051,7 +1042,7 @@ sub reindex_oid ($$$$) {
                        # do not delete from {mm_tmp}, since another
                        # single-MID message may use it.
                } else { # handle them at the end:
-                       multi_mid_q_push($sync, $oid);
+                       multi_mid_q_push($self, $sync, $oid);
                }
                return;
        }
@@ -1352,19 +1343,21 @@ sub index_sync {
        }
        if (my $multi_mid = delete $sync->{multi_mid}) {
                $git //= $self->{-inbox}->git;
-               my ($min, $max) = $multi_mid->minmax;
+               my $min = $multi_mid->{min};
+               my $max = $multi_mid->{max};
                if ($sync->{reindex}) {
                        # we may need to create new Message-IDs if mirrors
                        # were initially indexed with old versions
                        for (my $i = $max; $i >= $min; $i--) {
-                               my $oid = $multi_mid->mid_for($i);
+                               my $oid;
+                               $oid = $multi_mid->get_oid($i, $self) or next;
                                next unless defined $oid;
                                reindex_oid_m($self, $sync, $git, $oid);
                        }
                } else { # regen on initial index
                        for my $num ($min..$max) {
-                               my $oid = $multi_mid->mid_for($num);
-                               next unless defined $oid;
+                               my $oid;
+                               $oid = $multi_mid->get_oid($num, $self) or next;
                                reindex_oid_m($self, $sync, $git, $oid, $num);
                        }
                }