X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=script%2Fpublic-inbox-index;h=a52fb1bf399280218e8d01eff8a8e7bdb6d73d75;hb=99def67b86c4d270e8cfda4d1fad418291b6f3a4;hp=cf001cc1504e219cfe384168acd6a19f0a5cf3a1;hpb=d0e8bfd866ed1e924e8d9f551939eecbea4920ef;p=public-inbox.git diff --git a/script/public-inbox-index b/script/public-inbox-index index cf001cc1..a52fb1bf 100755 --- a/script/public-inbox-index +++ b/script/public-inbox-index @@ -1,110 +1,87 @@ #!/usr/bin/perl -w -# Copyright (C) 2015-2018 all contributors +# Copyright (C) 2015-2020 all contributors # License: AGPL-3.0+ -# Basic tool to create a Xapian search index for a git repository -# configured for public-inbox. +# Basic tool to create a Xapian search index for a public-inbox. # Usage with libeatmydata -# highly recommended: eatmydata public-inbox-index REPO_DIR +# highly recommended: eatmydata public-inbox-index INBOX_DIR use strict; use warnings; use Getopt::Long qw(:config gnu_getopt no_ignore_case auto_abbrev); -my $usage = "public-inbox-index REPO_DIR"; -use PublicInbox::Admin qw(resolve_repo_dir); +my $usage = "public-inbox-index INBOX_DIR"; +use PublicInbox::Admin; PublicInbox::Admin::require_or_die('-index'); -require PublicInbox::Config; +use PublicInbox::Xapcmd; -my $config = eval { PublicInbox::Config->new } || eval { - warn "public-inbox unconfigured for serving, indexing anyways...\n"; - undef; -}; +my $compact_opt; +my $opt = { quiet => -1, compact => 0, maxsize => undef, sync => 1 }; +GetOptions($opt, qw(verbose|v+ reindex rethread compact|c+ jobs|j=i prune sync! + xapianonly|xapian-only + indexlevel|L=s maxsize|max-size=s batchsize|batch-size=s + sequentialshard|seq-shard|sequential-shard)) + or die "bad command-line args\n$usage"; +die "--jobs must be >= 0\n" if defined $opt->{jobs} && $opt->{jobs} < 0; -my $reindex; -my $prune; -my $jobs = undef; -my $indexlevel; -my %opts = ( - '--reindex' => \$reindex, - '--jobs|j=i' => \$jobs, - '--prune' => \$prune, - 'L|indexlevel=s' => \$indexlevel, -); -GetOptions(%opts) or die "bad command-line args\n$usage"; -die "--jobs must be positive\n" if defined $jobs && $jobs < 0; +if ($opt->{compact}) { + require PublicInbox::Xapcmd; + PublicInbox::Xapcmd::check_compact(); + $compact_opt = { -coarse_lock => 1, compact => 1 }; + if (defined(my $jobs = $opt->{jobs})) { + $compact_opt->{jobs} = $jobs; + } +} -my @dirs; +my $cfg = PublicInbox::Config->new; +my @ibxs = PublicInbox::Admin::resolve_inboxes(\@ARGV, undef, $cfg); +PublicInbox::Admin::require_or_die('-index'); +unless (@ibxs) { print STDERR "Usage: $usage\n"; exit 1 } -if (@ARGV) { - @dirs = map { resolve_repo_dir($_) } @ARGV; -} else { - @dirs = (resolve_repo_dir()); +my $max_size = $opt->{maxsize} // $cfg->{lc('publicInbox.indexMaxSize')}; +if (defined $max_size) { + PublicInbox::Admin::parse_unsigned(\$max_size) or + die "`publicInbox.indexMaxSize=$max_size' not parsed\n"; } -sub usage { print STDERR "Usage: $usage\n"; exit 1 } -usage() unless @dirs; - -defined($config) and $config->each_inbox(sub { - my ($ibx) = @_; +if (my $bs = $opt->{batchsize} // $cfg->{lc('publicInbox.indexBatchSize')}) { + PublicInbox::Admin::parse_unsigned(\$bs) or + die "`publicInbox.indexBatchSize=$bs' not parsed\n"; + $PublicInbox::SearchIdx::BATCH_BYTES = $bs; +} - for my $i (0..$#dirs) { - next if $dirs[$i] ne $ibx->{mainrepo}; - $dirs[$i] = $ibx; - } -}); +my $s = $opt->{sequentialshard} // + $cfg->{lc('publicInbox.indexSequentialShard')}; +if (defined $s) { + my $v = $cfg->git_bool($s); + defined($v) or + die "`publicInbox.indexSequentialShard=$s' not boolean\n"; + $opt->{sequentialshard} = $v; +} -my @inboxes; my $mods = {}; - -foreach my $dir (@dirs) { - my $ibx = $dir; - if (!ref($ibx)) { - unless (-d $dir) { - die "$dir does not appear to be an inbox repository\n"; - } - $ibx = PublicInbox::Inbox->new({ - mainrepo => $dir, - name => 'unnamed', - indexlevel => $indexlevel, - version => -f "$dir/inbox.lock" ? 2 : 1, - }); - } elsif (defined $indexlevel && !defined($ibx->{indexlevel})) { - # XXX: users can shoot themselves in the foot, with this... - $ibx->{indexlevel} = $indexlevel; - } - push @inboxes, $ibx; +foreach my $ibx (@ibxs) { + # XXX: users can shoot themselves in the foot, with opt->{indexlevel} + $ibx->{indexlevel} //= $opt->{indexlevel} // ($opt->{xapianonly} ? + 'full' : PublicInbox::Admin::detect_indexlevel($ibx)); + $ibx->{index_max_size} = $max_size; PublicInbox::Admin::scan_ibx_modules($mods, $ibx); } PublicInbox::Admin::require_or_die(keys %$mods); +PublicInbox::Admin::progress_prepare($opt); +for my $ibx (@ibxs) { + $ibx = PublicInbox::InboxWritable->new($ibx); + if ($opt->{compact} >= 2) { + PublicInbox::Xapcmd::run($ibx, 'compact', $compact_opt); + } + $ibx->{-no_sync} = 1 if !$opt->{sync}; -require PublicInbox::SearchIdx; -index_inbox($_) for @inboxes; - -sub index_inbox { - my ($repo) = @_; - if (ref($repo) && ($repo->{version} || 1) == 2) { - eval { require PublicInbox::V2Writable }; - die "v2 requirements not met: $@\n" if $@; - my $v2w = eval { - PublicInbox::V2Writable->new($repo, {nproc=>$jobs}); - }; - if (defined $jobs) { - if ($jobs == 0) { - $v2w->{parallel} = 0; - } else { - my $n = $v2w->{partitions}; - if ($jobs != ($n + 1)) { - warn -"Unable to respect --jobs=$jobs, inbox was created with $n partitions\n"; - } - } - } - local $SIG{__WARN__} = sub { - print STDERR $v2w->{current_info}, ': ', @_; - }; - $v2w->index_sync({ reindex => $reindex, prune => $prune }); - } else { - my $s = PublicInbox::SearchIdx->new($repo, 1); - $s->index_sync({ reindex => $reindex }); + my $ibx_opt = $opt; + if (defined(my $s = $ibx->{indexsequentialshard})) { + defined(my $v = $cfg->git_bool($s)) or die <{name}.indexSequentialShard not boolean +EOL + $ibx_opt = { %$opt, sequentialshard => $v }; } + PublicInbox::Admin::index_inbox($ibx, undef, $ibx_opt); + PublicInbox::Xapcmd::run($ibx, 'compact', $compact_opt) if $compact_opt; }