use Carp qw(croak carp);
use Sys::Hostname qw(hostname);
use POSIX qw(strftime);
+use File::Glob qw(bsd_glob GLOB_NOSORT);
use PublicInbox::Search;
use PublicInbox::SearchIdx qw(prepare_stack is_ancestor is_bad_blob);
use PublicInbox::OverIdx;
use PublicInbox::InboxWritable;
use PublicInbox::ContentHash qw(content_hash);
use PublicInbox::Eml;
-use PublicInbox::DS qw(now);
+use PublicInbox::DS qw(now add_timer);
use DBI qw(:sql_types); # SQL_BLOB
sub new {
$self->idx_init($opt); # acquire lock via V2Writable::_idx_init
my $dbh = $self->{oidx}->dbh;
+ $dbh->do('PRAGMA case_sensitive_like = ON'); # only place we use LIKE
my $x3_doc = $dbh->prepare('SELECT docid FROM xref3 WHERE ibx_id = ?');
my $ibx_ck = $dbh->prepare('SELECT ibx_id,eidx_key FROM inboxes');
- my $lc_i = $dbh->prepare('SELECT key FROM eidx_meta WHERE key LIKE ?');
+ my $lc_i = $dbh->prepare(<<'');
+SELECT key FROM eidx_meta WHERE key LIKE ? ESCAPE ?
$ibx_ck->execute;
while (my ($ibx_id, $eidx_key) = $ibx_ck->fetchrow_array) {
# drop last_commit info
my $pat = $eidx_key;
- $pat =~ s/([_%])/\\$1/g;
- $lc_i->execute("lc-%:$pat//%");
+ $pat =~ s/([_%\\])/\\$1/g;
+ $lc_i->execute("lc-%:$pat//%", '\\');
while (my ($key) = $lc_i->fetchrow_array) {
next if $key !~ m!\Alc-v[1-9]+:\Q$eidx_key\E//!;
warn "I: removing $key\n";
sub _idx_init { # with_umask callback
my ($self, $opt) = @_;
- PublicInbox::V2Writable::_idx_init($self, $opt);
+ PublicInbox::V2Writable::_idx_init($self, $opt); # acquires ei.lock
$self->{midx} = PublicInbox::MiscIdx->new($self);
}
+sub symlink_packs ($$) {
+ my ($ibx, $pd) = @_;
+ my $ret = 0;
+ my $glob = "$ibx->{inboxdir}/git/*.git/objects/pack/*.idx";
+ for my $idx (bsd_glob($glob, GLOB_NOSORT)) {
+ my $src = substr($idx, 0, -length('.idx'));
+ my $dst = $pd . substr($src, rindex($src, '/'));
+ if (-f "$src.pack" and
+ symlink("$src.pack", "$dst.pack") and
+ symlink($idx, "$dst.idx") and
+ -f $idx) {
+ ++$ret;
+ # .promisor, .bitmap, .rev and .keep are optional
+ # XXX should we symlink .keep here?
+ for my $s (qw(promisor bitmap rev)) {
+ symlink("$src.$s", "$dst.$s") if -f "$src.$s";
+ }
+ } elsif (!$!{EEXIST}) {
+ warn "W: ln -s $src.{pack,idx} => $dst.*: $!\n";
+ unlink "$dst.pack", "$dst.idx";
+ }
+ }
+ $ret;
+}
+
sub idx_init { # similar to V2Writable
my ($self, $opt) = @_;
return if $self->{idx_shards};
}
}
}
+ # git-multi-pack-index(1) can speed up "git cat-file" startup slightly
+ my $dh;
+ my $git_midx = 0;
+ my $pd = "$ALL/objects/pack";
+ if (!mkdir($pd) && $!{EEXIST} && opendir($dh, $pd)) {
+ # drop stale symlinks
+ while (defined(my $dn = readdir($dh))) {
+ if ($dn =~ /\.(?:idx|pack|promisor|bitmap|rev)\z/) {
+ my $f = "$pd/$dn";
+ unlink($f) if -l $f && !-e $f;
+ }
+ }
+ undef $dh;
+ }
for my $ibx (@{$self->{ibx_list}}) {
+ # create symlinks for multi-pack-index
+ $git_midx += symlink_packs($ibx, $pd);
+ # add new lines to our alternates file
my $line = $ibx->git->{git_dir} . "/objects\n";
chomp(my $d = $line);
if (my @st = stat($d)) {
my $o = \@old;
PublicInbox::V2Writable::write_alternates($info_dir, $mode, $o);
}
+ $git_midx and $self->with_umask(sub {
+ my @cmd = ('multi-pack-index');
+ push @cmd, '--no-progress' if ($opt->{quiet}//0) > 1;
+ system('git', "--git-dir=$ALL", @cmd, 'write');
+ # ignore errors, fairly new command, may not exist
+ });
$self->parallel_init($self->{indexlevel});
$self->with_umask(\&_idx_init, $self, $opt);
$self->{oidx}->begin_lazy;
$self->{oidx}->eidx_prep;
- $self->git->batch_prepare;
- $self->{midx}->begin_txn;
+ $self->{midx}->create_xdb if @new;
}
sub _watch_commit { # PublicInbox::DS::add_timer callback
delete $self->{-commit_timer};
eidxq_process($self, $self->{-watch_sync});
eidxq_release($self);
- delete local $self->{-watch_sync}->{-regen_fmt};
+ my $fmt = delete $self->{-watch_sync}->{-regen_fmt};
reindex_checkpoint($self, $self->{-watch_sync});
+ $self->{-watch_sync}->{-regen_fmt} = $fmt;
# call event_step => done unless commit_timer is armed
PublicInbox::DS::requeue($self);
$pr->("indexing $ekey\n") if $pr;
$self->idx_init($opt);
sync_inbox($self, $self->{-watch_sync}, $ibx);
- $self->{-commit_timer} //= PublicInbox::DS::add_timer(
- $opt->{'commit-interval'} // 10,
+ $self->{-commit_timer} //= add_timer($opt->{'commit-interval'} // 10,
\&_watch_commit, $self);
}
*atfork_child = \&PublicInbox::V2Writable::atfork_child;
*idx_shard = \&PublicInbox::V2Writable::idx_shard;
*reindex_checkpoint = \&PublicInbox::V2Writable::reindex_checkpoint;
+*checkpoint = \&PublicInbox::V2Writable::checkpoint;
1;