Documentation/public-inbox-init.pod | 14 ++++++++++++++ script/public-inbox-init | 8 ++++++++ t/v2mirror.t | 4 +++- diff --git a/Documentation/public-inbox-init.pod b/Documentation/public-inbox-init.pod index 4744da96eb68c71d326e813ac037d1c743fbca54..495a258fdb696c3037d9adc7cf262f3cd39f68cc 100644 --- a/Documentation/public-inbox-init.pod +++ b/Documentation/public-inbox-init.pod @@ -48,6 +48,20 @@ added-after-the-fact (without affecting "git clone" followers). Default: unset, no epochs are skipped +=item -j, --jobs=JOBS + +Control the number of Xapian index shards in a +C<-V2> (L) inbox. + +It is useful to use a single shard (C<-j1>) for inboxes on +high-latency storage (e.g. rotational HDD) unless the system has +enough RAM to cache 5-10x the size of the git repository. + +It is generally not useful to specify higher values than the +default due to contention in the top-level producer process. + +Default: the number of online CPUs, up to 4 + =back =head1 ENVIRONMENT diff --git a/script/public-inbox-init b/script/public-inbox-init index 10d3ad45b2e6edb70ac51f809bb0b1d16d64a8b4..00147db557884cbc3d820da4ae05f96c7e8ee7e0 100755 --- a/script/public-inbox-init +++ b/script/public-inbox-init @@ -27,10 +27,12 @@ my $version = undef; my $indexlevel = undef; my $skip_epoch; +my $jobs; my %opts = ( 'V|version=i' => \$version, 'L|indexlevel=s' => \$indexlevel, 'S|skip|skip-epoch=i' => \$skip_epoch, + 'j|jobs=i' => \$jobs, ); GetOptions(%opts) or usage(); PublicInbox::Admin::indexlevel_ok_or_die($indexlevel) if defined $indexlevel; @@ -144,6 +146,12 @@ indexlevel => $indexlevel, }); my $creat_opt = {}; +if (defined $jobs) { + die "--jobs is only supported for -V2 inboxes\n" if $version == 1; + die "--jobs=$jobs must be >= 1\n" if $jobs <= 0; + $creat_opt->{nproc} = $jobs; +} + PublicInbox::InboxWritable->new($ibx, $creat_opt)->init_inbox(0, $skip_epoch); # needed for git prior to v2.1.0 diff --git a/t/v2mirror.t b/t/v2mirror.t index fc03c3d7c3d101730416afc2924335e8abbaeec1..b24528fee76307d7b8a17c7fb5873c6fbdd43e01 100644 --- a/t/v2mirror.t +++ b/t/v2mirror.t @@ -80,9 +80,11 @@ is(xsys(@cmd), 0, "cloned $i.git"); ok(-d "$tmpdir/m/git/$i.git", "mirror $i OK"); } -@cmd = ("-init", '-V2', 'm', "$tmpdir/m", 'http://example.com/m', +@cmd = ("-init", '-j1', '-V2', 'm', "$tmpdir/m", 'http://example.com/m', 'alt@example.com'); ok(run_script(\@cmd), 'initialized public-inbox -V2'); +my @shards = glob("$tmpdir/m/xap*/?"); +is(scalar(@shards), 1, 'got a single shard on init'); ok(run_script([qw(-index -j0), "$tmpdir/m"]), 'indexed');