use PublicInbox::Eml;
use PublicInbox::Git;
use PublicInbox::Import;
+use PublicInbox::MultiGit;
use PublicInbox::MID qw(mids references);
-use PublicInbox::ContentHash qw(content_hash content_digest);
+use PublicInbox::ContentHash qw(content_hash content_digest git_sha);
use PublicInbox::InboxWritable;
use PublicInbox::OverIdx;
use PublicInbox::Msgmap;
sub count_shards ($) {
my ($self) = @_;
+ # always load existing shards in case core count changes:
+ # Also, shard count may change while -watch is running
if (my $ibx = $self->{ibx}) {
- # always load existing shards in case core count changes:
- # Also, shard count may change while -watch is running
my $srch = $ibx->search or return 0;
delete $ibx->{search};
$srch->{nshard} // 0
} else { # ExtSearchIdx
- $self->{nshard} ||= scalar($self->xdb_shards_flat);
+ $self->{nshard} = scalar($self->xdb_shards_flat);
}
}
$v2ibx = PublicInbox::InboxWritable->new($v2ibx);
my $dir = $v2ibx->assert_usable_dir;
unless (-d $dir) {
- if ($creat) {
- require File::Path;
- File::Path::mkpath($dir);
- } else {
- die "$dir does not exist\n";
- }
+ die "$dir does not exist\n" if !$creat;
+ require File::Path;
+ File::Path::mkpath($dir);
}
my $xpfx = "$dir/xap" . PublicInbox::Search::SCHEMA_VERSION;
my $self = {
ibx => $v2ibx,
+ mg => PublicInbox::MultiGit->new($dir, 'all.git', 'git'),
im => undef, # PublicInbox::Import
parallel => 1,
transact_bytes => 0,
$self->{mm}->skip_artnum($skip_artnum) if defined $skip_artnum;
my $max = $self->{ibx}->max_git_epoch;
$max = $skip_epoch if (defined($skip_epoch) && !defined($max));
- $self->git_init($max // 0);
+ $self->{mg}->add_epoch($max // 0);
$self->done;
}
$rewritten->{rewrites}
}
-# returns the git object_id of $fh, does not write the object to FS
-sub git_hash_raw ($$) {
- my ($self, $raw) = @_;
- # grab the expected OID we have to reindex:
- pipe(my($in, $w)) or die "pipe: $!";
- my $git_dir = $self->git->{git_dir};
- my $cmd = ['git', "--git-dir=$git_dir", qw(hash-object --stdin)];
- my $r = popen_rd($cmd, undef, { 0 => $in });
- print $w $$raw or die "print \$w: $!";
- close $w or die "close \$w: $!";
- local $/ = "\n";
- chomp(my $oid = <$r>);
- close $r or die "git hash-object failed: $?";
- $oid =~ /\A$OID\z/ or die "OID not expected: $oid";
- $oid;
-}
-
sub _check_mids_match ($$$) {
my ($old_list, $new_list, $hdrs) = @_;
my %old_mids = map { $_ => 1 } @$old_list;
PublicInbox::Import::drop_unwanted_headers($new_mime);
my $raw = $new_mime->as_string;
- my $expect_oid = git_hash_raw($self, \$raw);
+ my $expect_oid = git_sha(1, \$raw)->hexdigest;
my $rewritten = _replace($self, $old_mime, $new_mime, \$raw) or return;
my $need_reindex = $rewritten->{need_reindex};
die $err if $err;
}
-sub write_alternates ($$$) {
- my ($info_dir, $mode, $out) = @_;
- my $fh = File::Temp->new(TEMPLATE => 'alt-XXXX', DIR => $info_dir);
- my $tmp = $fh->filename;
- print $fh @$out or die "print $tmp: $!\n";
- chmod($mode, $fh) or die "fchmod $tmp: $!\n";
- close $fh or die "close $tmp $!\n";
- my $alt = "$info_dir/alternates";
- rename($tmp, $alt) or die "rename $tmp => $alt: $!\n";
- $fh->unlink_on_destroy(0);
-}
-
-sub fill_alternates ($$) {
- my ($self, $epoch) = @_;
-
- my $pfx = "$self->{ibx}->{inboxdir}/git";
- my $all = "$self->{ibx}->{inboxdir}/all.git";
- PublicInbox::Import::init_bare($all) unless -d $all;
- my $info_dir = "$all/objects/info";
- my $alt = "$info_dir/alternates";
- my (%alt, $new);
- my $mode = 0644;
- if (-e $alt) {
- open(my $fh, '<', $alt) or die "open < $alt: $!\n";
- $mode = (stat($fh))[2] & 07777;
-
- # we assign a sort score to every alternate and favor
- # the newest (highest numbered) one because loose objects
- # require scanning epochs and only the latest epoch is
- # expected to see loose objects
- my $score;
- my $other = 0; # in case admin adds non-epoch repos
- %alt = map {;
- if (m!\A\Q../../\E([0-9]+)\.git/objects\z!) {
- $score = $1 + 0;
- } else {
- $score = --$other;
- }
- $_ => $score;
- } split(/\n+/, do { local $/; <$fh> });
- }
-
- foreach my $i (0..$epoch) {
- my $dir = "../../git/$i.git/objects";
- if (!exists($alt{$dir}) && -d "$pfx/$i.git") {
- $alt{$dir} = $i;
- $new = 1;
- }
- }
- return unless $new;
- write_alternates($info_dir, $mode,
- [join("\n", sort { $alt{$b} <=> $alt{$a} } keys %alt), "\n"]);
-}
-
-sub git_init {
- my ($self, $epoch) = @_;
- my $git_dir = "$self->{ibx}->{inboxdir}/git/$epoch.git";
- PublicInbox::Import::init_bare($git_dir);
- run_die([qw(git config), "--file=$git_dir/config",
- qw(include.path ../../all.git/config)]);
- fill_alternates($self, $epoch);
- $git_dir
-}
-
sub importer {
my ($self) = @_;
my $im = $self->{im};
$im->done;
$im = undef;
$self->checkpoint;
- my $git_dir = $self->git_init(++$self->{epoch_max});
- my $git = PublicInbox::Git->new($git_dir);
+ my $dir = $self->{mg}->add_epoch(++$self->{epoch_max});
+ my $git = PublicInbox::Git->new($dir);
return $self->import_init($git, 0);
}
}
}
}
$self->{epoch_max} = $epoch;
- $latest = $self->git_init($epoch);
- $self->import_init(PublicInbox::Git->new($latest), 0);
+ my $dir = $self->{mg}->add_epoch($epoch);
+ $self->import_init(PublicInbox::Git->new($dir), 0);
}
sub import_init {
chomp(my $n = $unit->{git}->qx(@cmd));
return if $n ne '' && $n == 0;
}
+ # don't rewind if --{since,until,before,after} are in use
+ return if (defined($last) &&
+ grep(defined, @{$sync->{-opt}}{qw(since until)}) &&
+ is_ancestor($self->git, $latest_cmt, $last));
+
last_epoch_commit($self, $unit->{epoch}, $latest_cmt);
}
sub xapian_only {
my ($self, $opt, $sync, $art_beg) = @_;
- my $seq = $opt->{sequential_shard};
+ my $seq = $opt->{'sequential-shard'};
$art_beg //= 0;
local $self->{parallel} = 0 if $seq;
$self->idx_init($opt); # acquire lock
}
my $pr = $opt->{-progress};
- my $seq = $opt->{sequential_shard};
+ my $seq = $opt->{'sequential-shard'};
my $art_beg; # the NNTP article number we start xapian_only at
my $idxlevel = $self->{ibx}->{indexlevel};
local $self->{ibx}->{indexlevel} = 'basic' if $seq;
$self->idx_init($opt); # acquire lock
- fill_alternates($self, $epoch_max);
+ $self->{mg}->fill_alternates;
$self->{oidx}->rethread_prepare($opt);
my $sync = {
need_checkpoint => \(my $bool = 0),
}
# reindex does not pick up new changes, so we rerun w/o it:
- if ($opt->{reindex} && !$sync->{quit}) {
+ if ($opt->{reindex} && !$sync->{quit} &&
+ !grep(defined, @$opt{qw(since until)})) {
my %again = %$opt;
$sync = undef;
delete @again{qw(rethread reindex -skip_lock)};
my ($self) = @_;
if (my $lei = delete $self->{lei}) {
$lei->_lei_atfork_child;
- close(delete $lei->{pkt_op_p});
+ my $pkt_op_p = delete $lei->{pkt_op_p};
+ close($pkt_op_p->{op_p});
}
$self->SUPER::ipc_atfork_child;
}