use PublicInbox::Eml;
use PublicInbox::Git;
use PublicInbox::Import;
+use PublicInbox::MultiGit;
use PublicInbox::MID qw(mids references);
use PublicInbox::ContentHash qw(content_hash content_digest git_sha);
use PublicInbox::InboxWritable;
$v2ibx = PublicInbox::InboxWritable->new($v2ibx);
my $dir = $v2ibx->assert_usable_dir;
unless (-d $dir) {
- if ($creat) {
- require File::Path;
- File::Path::mkpath($dir);
- } else {
- die "$dir does not exist\n";
- }
+ die "$dir does not exist\n" if !$creat;
+ require File::Path;
+ File::Path::mkpath($dir);
}
my $xpfx = "$dir/xap" . PublicInbox::Search::SCHEMA_VERSION;
my $self = {
ibx => $v2ibx,
+ mg => PublicInbox::MultiGit->new($dir, 'all.git', 'git'),
im => undef, # PublicInbox::Import
parallel => 1,
transact_bytes => 0,
$self->{mm}->skip_artnum($skip_artnum) if defined $skip_artnum;
my $max = $self->{ibx}->max_git_epoch;
$max = $skip_epoch if (defined($skip_epoch) && !defined($max));
- $self->git_init($max // 0);
+ $self->{mg}->add_epoch($max // 0);
$self->done;
}
# Now that all subprocesses are up, we can open the FDs
# for SQLite:
- my $mm = $self->{mm} = PublicInbox::Msgmap->new_file(
- "$ibx->{inboxdir}/msgmap.sqlite3",
- $ibx->{-no_fsync} ? 2 : 1);
+ my $mm = $self->{mm} = PublicInbox::Msgmap->new_file($ibx, 1);
$mm->{dbh}->begin_work;
}
}
my $shards = $self->{idx_shards};
if ($shards) {
- my $mm = $self->{mm};
- my $dbh = $mm->{dbh} if $mm;
+ my $dbh = $self->{mm}->{dbh} if $self->{mm};
# SQLite msgmap data is second in importance
$dbh->commit if $dbh;
+ eval { $dbh->do('PRAGMA optimize') };
# SQLite overview is third
$self->{oidx}->commit_lazy;
eval { $mm->{dbh}->$m };
$err .= "msgmap $m: $@\n" if $@;
}
+ if ($self->{oidx} && $self->{oidx}->{dbh} && $err) {
+ eval { $self->{oidx}->rollback_lazy };
+ $err .= "overview rollback: $@\n" if $@;
+ }
+
my $shards = delete $self->{idx_shards};
if ($shards) {
for (@$shards) {
die $err if $err;
}
-sub write_alternates ($$$) {
- my ($info_dir, $mode, $out) = @_;
- my $fh = File::Temp->new(TEMPLATE => 'alt-XXXX', DIR => $info_dir);
- my $tmp = $fh->filename;
- print $fh @$out or die "print $tmp: $!\n";
- chmod($mode, $fh) or die "fchmod $tmp: $!\n";
- close $fh or die "close $tmp $!\n";
- my $alt = "$info_dir/alternates";
- rename($tmp, $alt) or die "rename $tmp => $alt: $!\n";
- $fh->unlink_on_destroy(0);
-}
-
-sub fill_alternates ($$) {
- my ($self, $epoch) = @_;
-
- my $pfx = "$self->{ibx}->{inboxdir}/git";
- my $all = "$self->{ibx}->{inboxdir}/all.git";
- PublicInbox::Import::init_bare($all) unless -d $all;
- my $info_dir = "$all/objects/info";
- my $alt = "$info_dir/alternates";
- my (%alt, $new);
- my $mode = 0644;
- if (-e $alt) {
- open(my $fh, '<', $alt) or die "open < $alt: $!\n";
- $mode = (stat($fh))[2] & 07777;
-
- # we assign a sort score to every alternate and favor
- # the newest (highest numbered) one because loose objects
- # require scanning epochs and only the latest epoch is
- # expected to see loose objects
- my $score;
- my $other = 0; # in case admin adds non-epoch repos
- %alt = map {;
- if (m!\A\Q../../\E([0-9]+)\.git/objects\z!) {
- $score = $1 + 0;
- } else {
- $score = --$other;
- }
- $_ => $score;
- } split(/\n+/, do { local $/; <$fh> });
- }
-
- foreach my $i (0..$epoch) {
- my $dir = "../../git/$i.git/objects";
- if (!exists($alt{$dir}) && -d "$pfx/$i.git") {
- $alt{$dir} = $i;
- $new = 1;
- }
- }
- return unless $new;
- write_alternates($info_dir, $mode,
- [join("\n", sort { $alt{$b} <=> $alt{$a} } keys %alt), "\n"]);
-}
-
-sub git_init {
- my ($self, $epoch) = @_;
- my $git_dir = "$self->{ibx}->{inboxdir}/git/$epoch.git";
- PublicInbox::Import::init_bare($git_dir);
- run_die([qw(git config), "--file=$git_dir/config",
- qw(include.path ../../all.git/config)]);
- fill_alternates($self, $epoch);
- $git_dir
-}
-
sub importer {
my ($self) = @_;
my $im = $self->{im};
$im->done;
$im = undef;
$self->checkpoint;
- my $git_dir = $self->git_init(++$self->{epoch_max});
- my $git = PublicInbox::Git->new($git_dir);
+ my $dir = $self->{mg}->add_epoch(++$self->{epoch_max});
+ my $git = PublicInbox::Git->new($dir);
return $self->import_init($git, 0);
}
}
}
}
$self->{epoch_max} = $epoch;
- $latest = $self->git_init($epoch);
- $self->import_init(PublicInbox::Git->new($latest), 0);
+ my $dir = $self->{mg}->add_epoch($epoch);
+ $self->import_init(PublicInbox::Git->new($dir), 0);
}
sub import_init {
}
}
}
+ my $oidx = $self->{oidx};
if (!defined($num)) { # reuse if reindexing (or duplicates)
- my $oidx = $self->{oidx};
for my $mid (@$mids) {
($num, $mid0) = $oidx->num_mid0_for_oid($oid, $mid);
last if defined $num;
}
$mid0 //= do { # is this a number we got before?
$num = $arg->{mm_tmp}->num_for($mids->[0]);
+
+ # don't clobber existing if Message-ID is reused:
+ if (my $x = defined($num) ? $oidx->get_art($num) : undef) {
+ undef($num) if $x->{blob} ne $oid;
+ }
defined($num) ? $mids->[0] : undef;
};
if (!defined($num)) {
chomp(my $n = $unit->{git}->qx(@cmd));
return if $n ne '' && $n == 0;
}
+ # don't rewind if --{since,until,before,after} are in use
+ return if (defined($last) &&
+ grep(defined, @{$sync->{-opt}}{qw(since until)}) &&
+ is_ancestor($self->git, $latest_cmt, $last));
+
last_epoch_commit($self, $unit->{epoch}, $latest_cmt);
}
my $req = { %$sync, oid => $oid };
$self->git->cat_async($oid, $unindex_oid, $req);
}
- $self->git->cat_async_wait;
+ $self->git->async_wait_all;
}
return 0 if $sync->{quit};
if (!$regen_max) {
$self->git->cat_async($1, $unindex_oid, { %$sync, oid => $1 });
}
close $fh or die "git log failed: \$?=$?";
- $self->git->cat_async_wait;
+ $self->git->async_wait_all;
return unless $sync->{-opt}->{prune};
my $after = scalar keys %$unindexed;
index_xap_step($self, $sync, $art_beg, 1);
}
}
- $self->git->cat_async_wait;
+ $self->git->async_wait_all;
$self->{ibx}->cleanup;
$self->done;
}
local $self->{ibx}->{indexlevel} = 'basic' if $seq;
$self->idx_init($opt); # acquire lock
- fill_alternates($self, $epoch_max);
+ $self->{mg}->fill_alternates;
$self->{oidx}->rethread_prepare($opt);
my $sync = {
need_checkpoint => \(my $bool = 0),
}
# reindex does not pick up new changes, so we rerun w/o it:
- if ($opt->{reindex} && !$sync->{quit}) {
+ if ($opt->{reindex} && !$sync->{quit} &&
+ !grep(defined, @$opt{qw(since until)})) {
my %again = %$opt;
$sync = undef;
delete @again{qw(rethread reindex -skip_lock)};