-# Copyright (C) 2018 all contributors <meta@public-inbox.org>
+# Copyright (C) 2018-2019 all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
# This interface wraps and mimics PublicInbox::Import
my $PACKING_FACTOR = 0.4;
# SATA storage lags behind what CPUs are capable of, so relying on
-# nproc(1) can be misleading and having extra Xapian partions is a
+# nproc(1) can be misleading and having extra Xapian shards is a
# waste of FDs and space. It can also lead to excessive IO latency
# and slow things down. Users on NVME or other fast storage can
# use the NPROC env or switches in our script/public-inbox-* programs
my $n = 0;
my $xpfx = $self->{xpfx};
- # always load existing partitions in case core count changes:
+ # always load existing shards in case core count changes:
# Also, shard count may change while -watch is running
# due to "xcpdb --reshard"
if (-d $xpfx) {
# $creat may be any true value, or 0/undef. A hashref is true,
# and $creat->{nproc} may be set to an integer
my ($class, $v2ibx, $creat) = @_;
- my $dir = $v2ibx->{mainrepo} or die "no mainrepo in inbox\n";
+ my $dir = $v2ibx->{inboxdir} or die "no inboxdir in inbox\n";
unless (-d $dir) {
if ($creat) {
require File::Path;
$self->{shards} = $nshards;
}
- # need to create all parts before initializing msgmap FD
+ # need to create all shards before initializing msgmap FD
my $max = $self->{shards} - 1;
# idx_shards must be visible to all forked processes
# Now that all subprocesses are up, we can open the FDs
# for SQLite:
my $mm = $self->{mm} = PublicInbox::Msgmap->new_file(
- "$self->{-inbox}->{mainrepo}/msgmap.sqlite3", 1);
+ "$self->{-inbox}->{inboxdir}/msgmap.sqlite3", 1);
$mm->{dbh}->begin_work;
});
}
sub _replace_oids ($$$) {
my ($self, $mime, $replace_map) = @_;
$self->done;
- my $pfx = "$self->{-inbox}->{mainrepo}/git";
+ my $pfx = "$self->{-inbox}->{inboxdir}/git";
my $rewrites = []; # epoch => commit
my $max = $self->{epoch_max};
$rewritten->{rewrites};
}
-sub last_commit_part ($$;$) {
+sub last_epoch_commit ($$;$) {
my ($self, $i, $cmt) = @_;
my $v = PublicInbox::Search::SCHEMA_VERSION();
$self->{mm}->last_commit_xap($v, $i, $cmt);
foreach my $i (0..$epoch_max) {
defined(my $cmt = $last_commit->[$i]) or next;
$last_commit->[$i] = undef;
- last_commit_part($self, $i, $cmt);
+ last_epoch_commit($self, $i, $cmt);
}
}
if ($wait) {
my $barrier = $self->barrier_init(scalar @$shards);
- # each partition needs to issue a barrier command
+ # each shard needs to issue a barrier command
$_->remote_barrier for @$shards;
- # wait for each Xapian partition
+ # wait for each Xapian shard
$self->barrier_wait($barrier);
} else {
$_->remote_commit for @$shards;
}
# last_commit is special, don't commit these until
- # remote partitions are done:
+ # remote shards are done:
$dbh->begin_work;
set_last_commits($self);
$dbh->commit;
sub fill_alternates ($$) {
my ($self, $epoch) = @_;
- my $pfx = "$self->{-inbox}->{mainrepo}/git";
- my $all = "$self->{-inbox}->{mainrepo}/all.git";
- my @cmd;
+ my $pfx = "$self->{-inbox}->{inboxdir}/git";
+ my $all = "$self->{-inbox}->{inboxdir}/all.git";
+
unless (-d $all) {
PublicInbox::Import::init_bare($all);
}
- @cmd = (qw/git config/, "--file=$pfx/$epoch.git/config",
- 'include.path', '../../all.git/config');
- PublicInbox::Import::run_die(\@cmd);
-
my $alt = "$all/objects/info/alternates";
my %alts;
my @add;
sub git_init {
my ($self, $epoch) = @_;
- my $git_dir = "$self->{-inbox}->{mainrepo}/git/$epoch.git";
+ my $git_dir = "$self->{-inbox}->{inboxdir}/git/$epoch.git";
my @cmd = (qw(git init --bare -q), $git_dir);
PublicInbox::Import::run_die(\@cmd);
+ @cmd = (qw/git config/, "--file=$git_dir/config",
+ 'include.path', '../../all.git/config');
+ PublicInbox::Import::run_die(\@cmd);
fill_alternates($self, $epoch);
$git_dir
}
sub git_dir_latest {
my ($self, $max) = @_;
$$max = -1;
- my $pfx = "$self->{-inbox}->{mainrepo}/git";
+ my $pfx = "$self->{-inbox}->{inboxdir}/git";
return unless -d $pfx;
my $latest;
opendir my $dh, $pfx or die "opendir $pfx: $!\n";
# only update last_commit for $i on reindex iff newer than current
sub update_last_commit ($$$$) {
my ($self, $git, $i, $cmt) = @_;
- my $last = last_commit_part($self, $i);
+ my $last = last_epoch_commit($self, $i);
if (defined $last && is_ancestor($git, $last, $cmt)) {
my @cmd = (qw(rev-list --count), "$last..$cmt");
chomp(my $n = $git->qx(@cmd));
return if $n ne '' && $n == 0;
}
- last_commit_part($self, $i, $cmt);
+ last_epoch_commit($self, $i, $cmt);
}
-sub git_dir_n ($$) { "$_[0]->{-inbox}->{mainrepo}/git/$_[1].git" }
+sub git_dir_n ($$) { "$_[0]->{-inbox}->{inboxdir}/git/$_[1].git" }
sub last_commits ($$) {
my ($self, $epoch_max) = @_;
my $heads = [];
for (my $i = $epoch_max; $i >= 0; $i--) {
- $heads->[$i] = last_commit_part($self, $i);
+ $heads->[$i] = last_epoch_commit($self, $i);
}
$heads;
}
for (my $i = $epoch_max; $i >= 0; $i--) {
die 'BUG: already indexing!' if $self->{reindex_pipe};
my $git_dir = git_dir_n($self, $i);
- -d $git_dir or next; # missing parts are fine
+ -d $git_dir or next; # missing epochs are fine
my $git = PublicInbox::Git->new($git_dir);
if ($reindex_heads) {
$head = $reindex_heads->[$i] or next;
join(',',sort keys %gone), "\n";
}
foreach my $num (keys %gone) {
- $self->{unindexed}->{$_}++;
+ $self->{unindexed}->{$num}++;
$self->{mm}->num_delete($num);
}
unindex_oid_remote($self, $oid, $mid);
my $git_dir = git_dir_n($self, $i);
die 'BUG: already reindexing!' if $self->{reindex_pipe};
- -d $git_dir or return; # missing parts are fine
+ -d $git_dir or return; # missing epochs are fine
fill_alternates($self, $i);
my $git = PublicInbox::Git->new($git_dir);
if (my $unindex_range = delete $sync->{unindex_range}->{$i}) {