my $PACKING_FACTOR = 0.4;
# assume 2 cores if GNU nproc(1) is not available
-sub nproc_parts () {
+sub nproc_parts ($) {
+ my ($creat_opt) = @_;
+ if (ref($creat_opt) eq 'HASH') {
+ if (defined(my $n = $creat_opt->{nproc})) {
+ return $n
+ }
+ }
+
my $n = int($ENV{NPROC} || `nproc 2>/dev/null` || 2);
# subtract for the main process and git-fast-import
$n -= 1;
}
sub new {
+ # $creat may be any true value, or 0/undef. A hashref is true,
+ # and $creat->{nproc} may be set to an integer
my ($class, $v2ibx, $creat) = @_;
my $dir = $v2ibx->{mainrepo} or die "no mainrepo in inbox\n";
unless (-d $dir) {
rotate_bytes => int((1024 * 1024 * 1024) / $PACKING_FACTOR),
last_commit => [], # git repo -> commit
};
- $self->{partitions} = count_partitions($self) || nproc_parts();
+ $self->{partitions} = count_partitions($self) || nproc_parts($creat);
bless $self, $class;
}
sub init_inbox {
- my ($self, $parallel, $skip) = @_;
+ my ($self, $parallel, $skip_epoch) = @_;
$self->{parallel} = $parallel;
$self->idx_init;
my $epoch_max = -1;
git_dir_latest($self, \$epoch_max);
- if (defined $skip && $epoch_max == -1) {
- $epoch_max = $skip;
+ if (defined $skip_epoch && $epoch_max == -1) {
+ $epoch_max = $skip_epoch;
}
$self->git_init($epoch_max >= 0 ? $epoch_max : 0);
$self->done;
my $all = "$self->{-inbox}->{mainrepo}/all.git";
my @cmd;
unless (-d $all) {
- @cmd = (qw(git init --bare -q), $all);
- PublicInbox::Import::run_die(\@cmd);
- @cmd = (qw/git config/, "--file=$all/config",
- 'repack.writeBitmaps', 'true');
- PublicInbox::Import::run_die(\@cmd);
+ PublicInbox::Import::init_bare($all);
}
@cmd = (qw/git config/, "--file=$pfx/$epoch.git/config",
'include.path', '../../all.git/config');
*is_ancestor = *PublicInbox::SearchIdx::is_ancestor;
+# returns a revision range for git-log(1)
+sub log_range ($$$$$) {
+ my ($self, $git, $ranges, $i, $tip) = @_;
+ my $cur = $ranges->[$i] or return $tip; # all of it
+ my $range = "$cur..$tip";
+ if (is_ancestor($git, $cur, $tip)) { # common case
+ my $n = $git->qx(qw(rev-list --count), $range);
+ chomp($n);
+ if ($n == 0) {
+ $ranges->[$i] = undef;
+ return; # nothing to do
+ }
+ } else {
+ warn <<"";
+discontiguous range: $range
+Rewritten history? (in $git->{git_dir})
+
+ chomp(my $base = $git->qx('merge-base', $tip, $cur));
+ if ($base) {
+ $range = "$base..$tip";
+ warn "found merge-base: $base\n"
+ } else {
+ $range = $tip;
+ warn "discarding history at $cur\n";
+ }
+ warn <<"";
+reindexing $git->{git_dir} starting at
+$range
+
+ $self->{"unindex-range.$i"} = "$base..$cur";
+ }
+ $range;
+}
+
sub index_prepare {
my ($self, $opts, $epoch_max, $ranges) = @_;
my $regen_max = 0;
-d $git_dir or next; # missing parts are fine
my $git = PublicInbox::Git->new($git_dir);
chomp(my $tip = $git->qx(qw(rev-parse -q --verify), $head));
- next if $?; # new repo
- my $range;
- if (defined(my $cur = $ranges->[$i])) {
- $range = "$cur..$tip";
- if (is_ancestor($git, $cur, $tip)) { # common case
- my $n = $git->qx(qw(rev-list --count), $range);
- chomp($n);
- if ($n == 0) {
- $ranges->[$i] = undef;
- next;
- }
- } else {
- warn <<"";
-discontiguous range: $range
-Rewritten history? (in $git_dir)
-
- my $base = $git->qx('merge-base', $tip, $cur);
- chomp $base;
- if ($base) {
- $range = "$base..$tip";
- warn "found merge-base: $base\n"
- } else {
- $range = $tip;
- warn <<"";
-discarding history at $cur
-
- }
- warn <<"";
-reindexing $git_dir starting at
-$range
- $self->{"unindex-range.$i"} = "$base..$cur";
- }
- } else {
- $range = $tip; # all of it
- }
+ next if $?; # new repo
+ my $range = log_range($self, $git, $ranges, $i, $tip) or next;
$ranges->[$i] = $range;
# can't use 'rev-list --count' if we use --diff-filter
qw(-c gc.reflogExpire=now gc --prune=all)]);
}
+# called for public-inbox-index
sub index_sync {
my ($self, $opts) = @_;
$opts ||= {};