Documentation/public-inbox-index.pod | 4 ++-- script/public-inbox-index | 44 ++++++++++++++++++++++++++++++++++++-------- diff --git a/Documentation/public-inbox-index.pod b/Documentation/public-inbox-index.pod index a4edc57a08bf6d0107f491437392a4282c5ef086..56dec99301ac2d256959c24780415a3a73885d74 100644 --- a/Documentation/public-inbox-index.pod +++ b/Documentation/public-inbox-index.pod @@ -40,8 +40,8 @@ C<--jobs=0> is accepted as of public-inbox 1.6.0 (PENDING) to disable parallel indexing. -If the inbox has not been indexed, C shards -will be created (one job is always needed for indexing +If the inbox has not been indexed or initialized, C +shards will be created (one job is always needed for indexing the overview and article number mapping). Default: the number of existing Xapian shards diff --git a/script/public-inbox-index b/script/public-inbox-index index e2bca16e92510221a0f95998277be6351dcac6f8..73ca29537b4feae1d44e1f636d115ac00fdb2d9f 100755 --- a/script/public-inbox-index +++ b/script/public-inbox-index @@ -1,4 +1,4 @@ -#!/usr/bin/perl -w +#!perl -w # Copyright (C) 2015-2020 all contributors # License: AGPL-3.0+ # Basic tool to create a Xapian search index for a public-inbox. @@ -6,21 +6,46 @@ # Usage with libeatmydata # highly recommended: eatmydata public-inbox-index INBOX_DIR use strict; -use warnings; +use v5.10.1; use Getopt::Long qw(:config gnu_getopt no_ignore_case auto_abbrev); -my $usage = "public-inbox-index INBOX_DIR"; -use PublicInbox::Admin; -PublicInbox::Admin::require_or_die('-index'); -use PublicInbox::Xapcmd; +my $usage = 'public-inbox-index [options] INBOX_DIR'; +my $help = < -1, compact => 0, maxsize => undef, fsync => 1 }; GetOptions($opt, qw(verbose|v+ reindex rethread compact|c+ jobs|j=i prune fsync|sync! xapianonly|xapian-only indexlevel|L=s maxsize|max-size=s batchsize|batch-size=s - sequentialshard|seq-shard|sequential-shard)) + sequentialshard|seq-shard|sequential-shard + help|?)) or die "bad command-line args\n$usage"; +if ($opt->{help}) { print $help; exit 0 }; die "--jobs must be >= 0\n" if defined $opt->{jobs} && $opt->{jobs} < 0; + +# require lazily to speed up --help +require PublicInbox::Admin; +PublicInbox::Admin::require_or_die('-index'); if ($opt->{compact}) { require PublicInbox::Xapcmd; @@ -31,7 +56,7 @@ $compact_opt->{jobs} = $jobs; } } -my $cfg = PublicInbox::Config->new; +my $cfg = PublicInbox::Config->new; # Config is loaded by Admin my @ibxs = PublicInbox::Admin::resolve_inboxes(\@ARGV, undef, $cfg); PublicInbox::Admin::require_or_die('-index'); unless (@ibxs) { print STDERR "Usage: $usage\n"; exit 1 } @@ -47,7 +72,9 @@ if (defined $bs) { PublicInbox::Admin::parse_unsigned(\$bs) or die "`publicInbox.indexBatchSize=$bs' not parsed\n"; } +no warnings 'once'; local $PublicInbox::SearchIdx::BATCH_BYTES = $bs if defined($bs); +use warnings 'once'; # out-of-the-box builds of Xapian 1.4.x are still limited to 32-bit # https://getting-started-with-xapian.readthedocs.io/en/latest/concepts/indexing/limitations.html @@ -72,6 +99,7 @@ PublicInbox::Admin::scan_ibx_modules($mods, $ibx); } PublicInbox::Admin::require_or_die(keys %$mods); +require PublicInbox::InboxWritable; PublicInbox::Admin::progress_prepare($opt); for my $ibx (@ibxs) { $ibx = PublicInbox::InboxWritable->new($ibx);