use Carp qw(croak carp);
use Sys::Hostname qw(hostname);
use POSIX qw(strftime);
+use File::Glob qw(bsd_glob GLOB_NOSORT);
use PublicInbox::Search;
use PublicInbox::SearchIdx qw(prepare_stack is_ancestor is_bad_blob);
use PublicInbox::OverIdx;
sub do_finalize ($) {
my ($req) = @_;
- if (my $indexed = $req->{indexed}) {
+ if (my $indexed = $req->{indexed}) { # duplicated messages
do_xpost($req, $_) for @$indexed;
} elsif (exists $req->{new_smsg}) { # totally unseen messsage
index_unseen($req);
\&ck_existing, $req);
return; # ck_existing calls do_step
}
- delete $req->{cur_smsg};
delete $req->{next_arg};
}
- my $mid = shift(@{$req->{mids}});
- last unless defined $mid;
+ die "BUG: {cur_smsg} still set" if $req->{cur_smsg};
+ my $mid = shift(@{$req->{mids}}) // last;
my ($id, $prev);
$req->{next_arg} = [ $mid, \$id, \$prev ];
# loop again
do_finalize($req);
}
-sub _blob_missing ($) { # called when req->{cur_smsg}->{blob} is bad
- my ($req) = @_;
- my $smsg = $req->{cur_smsg} or die 'BUG: {cur_smsg} missing';
+sub _blob_missing ($$) { # called when $smsg->{blob} is bad
+ my ($req, $smsg) = @_;
my $self = $req->{self};
my $xref3 = $self->{oidx}->get_xref3($smsg->{num});
my @keep = grep(!/:$smsg->{blob}\z/, @$xref3);
sub ck_existing { # git->cat_async callback
my ($bref, $oid, $type, $size, $req) = @_;
- my $smsg = $req->{cur_smsg} or die 'BUG: {cur_smsg} missing';
+ my $smsg = delete $req->{cur_smsg} or die 'BUG: {cur_smsg} missing';
if ($type eq 'missing') {
- _blob_missing($req);
+ _blob_missing($req, $smsg);
} elsif (!is_bad_blob($oid, $type, $size, $smsg->{blob})) {
my $self = $req->{self} // die 'BUG: {self} missing';
local $self->{current_info} = "$self->{current_info} $oid";
$req->{eml} = PublicInbox::Eml->new($bref);
$req->{chash} = content_hash($req->{eml});
$req->{mids} = mids($req->{eml});
- my @q = @{$req->{mids}}; # copy
- while (defined(my $mid = shift @q)) {
+ for my $mid (@{$req->{mids}}) {
my ($id, $prev);
while (my $x = $ibx->over->next_by_mid($mid, \$id, \$prev)) {
return $x->{num} if $x->{blob} eq $req->{oid};
$self->idx_init($opt); # acquire lock via V2Writable::_idx_init
my $dbh = $self->{oidx}->dbh;
+ $dbh->do('PRAGMA case_sensitive_like = ON'); # only place we use LIKE
my $x3_doc = $dbh->prepare('SELECT docid FROM xref3 WHERE ibx_id = ?');
my $ibx_ck = $dbh->prepare('SELECT ibx_id,eidx_key FROM inboxes');
- my $lc_i = $dbh->prepare('SELECT key FROM eidx_meta WHERE key LIKE ?');
+ my $lc_i = $dbh->prepare(<<'');
+SELECT key FROM eidx_meta WHERE key LIKE ? ESCAPE ?
$ibx_ck->execute;
while (my ($ibx_id, $eidx_key) = $ibx_ck->fetchrow_array) {
# drop last_commit info
my $pat = $eidx_key;
- $pat =~ s/([_%])/\\$1/g;
- $lc_i->execute("lc-%:$pat//%");
+ $pat =~ s/([_%\\])/\\$1/g;
+ $lc_i->execute("lc-%:$pat//%", '\\');
while (my ($key) = $lc_i->fetchrow_array) {
next if $key !~ m!\Alc-v[1-9]+:\Q$eidx_key\E//!;
warn "I: removing $key\n";
warn $err, "\n" if defined($err);
}
+sub dd_smsg { # git->cat_async callback
+ my ($bref, $oid, $type, $size, $dd) = @_;
+ my $smsg = $dd->{smsg} // die 'BUG: dd->{smsg} missing';
+ my $self = $dd->{self} // die 'BUG: {self} missing';
+ my $per_mid = $dd->{per_mid} // die 'BUG: {per_mid} missing';
+ if ($type eq 'missing') {
+ _blob_missing($dd, $smsg);
+ } elsif (!is_bad_blob($oid, $type, $size, $smsg->{blob})) {
+ local $self->{current_info} = "$self->{current_info} $oid";
+ my $chash = content_hash(PublicInbox::Eml->new($bref));
+ push(@{$per_mid->{dd_chash}->{$chash}}, $smsg);
+ }
+ return if $per_mid->{last_smsg} != $smsg;
+ while (my ($chash, $ary) = each %{$per_mid->{dd_chash}}) {
+ my $keep = shift @$ary;
+ next if !scalar(@$ary);
+ $per_mid->{sync}->{dedupe_cull} += scalar(@$ary);
+ print STDERR
+ "# <$keep->{mid}> keeping #$keep->{num}, dropping ",
+ join(', ', map { "#$_->{num}" } @$ary),"\n";
+ next if $per_mid->{sync}->{-opt}->{'dry-run'};
+ my $oidx = $self->{oidx};
+ for my $smsg (@$ary) {
+ my $gone = $smsg->{num};
+ $oidx->merge_xref3($keep->{num}, $gone, $smsg->{blob});
+ $self->idx_shard($gone)->ipc_do('xdb_remove', $gone);
+ $oidx->delete_by_num($gone);
+ }
+ }
+}
+
+sub eidx_dedupe ($$) {
+ my ($self, $sync) = @_;
+ $sync->{dedupe_cull} = 0;
+ my $candidates = 0;
+ my $nr_mid = 0;
+ return unless eidxq_lock_acquire($self);
+ my $iter;
+ my $min_id = 0;
+ local $sync->{-regen_fmt} = "dedupe %u/".$self->{oidx}->max."\n";
+dedupe_restart:
+ $iter = $self->{oidx}->dbh->prepare(<<EOS);
+SELECT DISTINCT(mid),id FROM msgid WHERE id IN
+(SELECT id FROM id2num WHERE id > ? GROUP BY num HAVING COUNT(num) > 1)
+ORDER BY id
+EOS
+ $iter->execute($min_id);
+ while (my ($mid, $id) = $iter->fetchrow_array) {
+ last if $sync->{quit};
+ $self->{current_info} = "dedupe $mid";
+ ${$sync->{nr}} = $min_id = $id;
+ my ($n, $prv, @smsg);
+ while (my $x = $self->{oidx}->next_by_mid($mid, \$n, \$prv)) {
+ push @smsg, $x;
+ }
+ next if scalar(@smsg) < 2;
+ my $per_mid = {
+ dd_chash => {}, # chash => [ary of smsgs]
+ last_smsg => $smsg[-1],
+ sync => $sync
+ };
+ $nr_mid++;
+ $candidates += scalar(@smsg) - 1;
+ for my $smsg (@smsg) {
+ my $dd = {
+ per_mid => $per_mid,
+ smsg => $smsg,
+ self => $self,
+ };
+ $self->git->cat_async($smsg->{blob}, \&dd_smsg, $dd);
+ }
+ # need to wait on every single one
+ $self->git->async_wait_all;
+
+ # is checkpoint needed? $iter is a very expensive query to restart
+ if (0 && checkpoint_due($sync)) {
+ undef $iter;
+ reindex_checkpoint($self, $sync);
+ goto dedupe_restart;
+ }
+ }
+ my $n = delete $sync->{dedupe_cull};
+ if (my $pr = $sync->{-opt}->{-progress}) {
+ $pr->("culled $n/$candidates candidates ($nr_mid msgids)\n");
+ }
+ ${$sync->{nr}} = 0;
+}
+
sub eidx_sync { # main entry point
my ($self, $opt) = @_;
my $warn_cb = $SIG{__WARN__} || \&CORE::warn;
local $self->{current_info} = '';
local $SIG{__WARN__} = sub {
+ return if PublicInbox::Eml::warn_ignore(@_);
$warn_cb->($self->{current_info}, ': ', @_);
};
$self->idx_init($opt); # acquire lock via V2Writable::_idx_init
for my $ibx (@{$self->{ibx_list}}) {
$ibx->{-ibx_id} //= $self->{oidx}->ibx_id($ibx->eidx_key);
}
+ if (delete($opt->{dedupe})) {
+ local $sync->{checkpoint_unlocks} = 1;
+ eidx_dedupe($self, $sync);
+ }
if (delete($opt->{reindex})) {
local $sync->{checkpoint_unlocks} = 1;
eidx_reindex($self, $sync);
sub _idx_init { # with_umask callback
my ($self, $opt) = @_;
- PublicInbox::V2Writable::_idx_init($self, $opt);
+ PublicInbox::V2Writable::_idx_init($self, $opt); # acquires ei.lock
$self->{midx} = PublicInbox::MiscIdx->new($self);
}
+sub symlink_packs ($$) {
+ my ($ibx, $pd) = @_;
+ my $ret = 0;
+ my $glob = "$ibx->{inboxdir}/git/*.git/objects/pack/*.idx";
+ for my $idx (bsd_glob($glob, GLOB_NOSORT)) {
+ my $src = substr($idx, 0, -length('.idx'));
+ my $dst = $pd . substr($src, rindex($src, '/'));
+ if (-f "$src.pack" and
+ symlink("$src.pack", "$dst.pack") and
+ symlink($idx, "$dst.idx") and
+ -f $idx) {
+ ++$ret;
+ # .promisor, .bitmap, .rev and .keep are optional
+ # XXX should we symlink .keep here?
+ for my $s (qw(promisor bitmap rev)) {
+ symlink("$src.$s", "$dst.$s") if -f "$src.$s";
+ }
+ } elsif (!$!{EEXIST}) {
+ warn "W: ln -s $src.{pack,idx} => $dst.*: $!\n";
+ unlink "$dst.pack", "$dst.idx";
+ }
+ }
+ $ret;
+}
+
sub idx_init { # similar to V2Writable
my ($self, $opt) = @_;
return if $self->{idx_shards};
}
}
}
+ # git-multi-pack-index(1) can speed up "git cat-file" startup slightly
+ my $dh;
+ my $git_midx = 0;
+ my $pd = "$ALL/objects/pack";
+ if (!mkdir($pd) && $!{EEXIST} && opendir($dh, $pd)) {
+ # drop stale symlinks
+ while (defined(my $dn = readdir($dh))) {
+ if ($dn =~ /\.(?:idx|pack|promisor|bitmap|rev)\z/) {
+ my $f = "$pd/$dn";
+ unlink($f) if -l $f && !-e $f;
+ }
+ }
+ undef $dh;
+ }
for my $ibx (@{$self->{ibx_list}}) {
+ # create symlinks for multi-pack-index
+ $git_midx += symlink_packs($ibx, $pd);
+ # add new lines to our alternates file
my $line = $ibx->git->{git_dir} . "/objects\n";
chomp(my $d = $line);
if (my @st = stat($d)) {
my $o = \@old;
PublicInbox::V2Writable::write_alternates($info_dir, $mode, $o);
}
+ $git_midx and $self->with_umask(sub {
+ my @cmd = ('multi-pack-index');
+ push @cmd, '--no-progress' if ($opt->{quiet}//0) > 1;
+ my $lk = $self->lock_for_scope;
+ system('git', "--git-dir=$ALL", @cmd, 'write');
+ # ignore errors, fairly new command, may not exist
+ });
$self->parallel_init($self->{indexlevel});
$self->with_umask(\&_idx_init, $self, $opt);
$self->{oidx}->begin_lazy;
my $oldset = PublicInbox::DS::block_signals();
local $self->{current_info} = '';
my $cb = $SIG{__WARN__} || \&CORE::warn;
- local $SIG{__WARN__} = sub { $cb->($self->{current_info}, ': ', @_) };
+ local $SIG{__WARN__} = sub {
+ return if PublicInbox::Eml::warn_ignore(@_);
+ $cb->($self->{current_info}, ': ', @_);
+ };
my $sig = {
HUP => sub { eidx_reload($self, $idler) },
USR1 => sub { eidx_resync_start($self) },