# This writes to the search index.
package PublicInbox::SearchIdx;
use strict;
-use warnings;
-use base qw(PublicInbox::Search PublicInbox::Lock);
+use v5.10.1;
+use parent qw(PublicInbox::Search PublicInbox::Lock);
use PublicInbox::Eml;
use PublicInbox::InboxWritable;
-use PublicInbox::MID qw/mid_clean mid_mime mids_for_index/;
+use PublicInbox::MID qw(mid_mime mids_for_index mids);
use PublicInbox::MsgIter;
use Carp qw(croak);
use POSIX qw(strftime);
$ibx = PublicInbox::InboxWritable->new($ibx);
my $self = bless {
inboxdir => $inboxdir,
- -inbox => $ibx,
- git => $ibx->git,
+ ibx => $ibx,
-altid => $altid,
ibx_ver => $version,
indexlevel => $indexlevel,
sub need_xapian ($) { $_[0]->{indexlevel} =~ $xapianlevels }
sub _xdb_release {
- my ($self) = @_;
+ my ($self, $wake) = @_;
if (need_xapian($self)) {
my $xdb = delete $self->{xdb} or croak 'not acquired';
$xdb->close;
}
- $self->lock_release if $self->{creat};
+ $self->lock_release($wake) if $self->{creat};
undef;
}
add_val($doc, PublicInbox::Search::YYYYMMDD(), $yyyymmdd);
my $dt = strftime('%Y%m%d%H%M%S', @ds);
add_val($doc, PublicInbox::Search::DT(), $dt);
+ add_val($doc, PublicInbox::Search::BYTES(), $smsg->{bytes});
+ add_val($doc, PublicInbox::Search::UID(), $smsg->{num});
my $tg = term_generator($self);
$tg->set_document($doc);
sub add_message {
# mime = PublicInbox::Eml or Email::MIME object
- my ($self, $mime, $smsg) = @_;
+ my ($self, $mime, $smsg, $sync) = @_;
my $hdr = $mime->header_obj;
my $mids = mids_for_index($hdr);
$smsg //= bless { blob => '' }, 'PublicInbox::Smsg'; # test-only compat
};
# v1 and tests only:
- $smsg->populate($hdr, $self);
+ $smsg->populate($hdr, $sync);
+ $smsg->{bytes} //= length($mime->as_string);
eval {
# order matters, overview stores every possible piece of
$smsg->{num};
}
-# returns begin and end PostingIterator
-sub find_doc_ids {
- my ($self, $termval) = @_;
- my $db = $self->{xdb};
-
- ($db->postlist_begin($termval), $db->postlist_end($termval));
-}
-
-# v1 only
-sub batch_do {
- my ($self, $termval, $cb) = @_;
- my $batch_size = 1000; # don't let @ids grow too large to avoid OOM
- while (1) {
- my ($head, $tail) = $self->find_doc_ids($termval);
- return if $head == $tail;
- my @ids;
- for (; $head != $tail && @ids < $batch_size; $head++) {
- push @ids, $head->get_docid;
+sub xdb_remove {
+ my ($self, $oid, @removed) = @_;
+ my $xdb = $self->{xdb} or return;
+ for my $num (@removed) {
+ my $doc = eval { $xdb->get_document($num) };
+ unless ($doc) {
+ warn "E: $@\n" if $@;
+ warn "E: #$num $oid missing in Xapian\n";
+ next;
}
- $cb->(\@ids);
- }
-}
-
-# v1 only, where $mid is unique
-sub remove_message {
- my ($self, $mid) = @_;
- $mid = mid_clean($mid);
-
- if (my $over = $self->{over}) {
- my $nr = eval { $over->remove_oid(undef, $mid) };
- if ($@) {
- warn "failed to remove <$mid> from overview: $@\n";
- } elsif ($nr == 0) {
- warn "<$mid> missing for removal from overview\n";
+ my $smsg = bless {}, 'PublicInbox::Smsg';
+ $smsg->load_expand($doc);
+ my $blob = $smsg->{blob} // '(unset)';
+ if ($blob eq $oid) {
+ $xdb->delete_document($num);
+ } else {
+ warn "E: #$num $oid != $blob in Xapian\n";
}
}
- return unless need_xapian($self);
- my $db = $self->{xdb};
- my $nr = 0;
- eval {
- batch_do($self, 'Q' . $mid, sub {
- my ($ids) = @_;
- $db->delete_document($_) for @$ids;
- $nr += scalar @$ids;
- });
- };
- if ($@) {
- warn "failed to remove <$mid> from Xapian: $@\n";
- } elsif ($nr == 0) {
- warn "<$mid> missing for removal from Xapian\n";
- }
}
-# MID is a hint in V2
sub remove_by_oid {
- my ($self, $oid, $mid) = @_;
-
- $self->{over}->remove_oid($oid, $mid) if $self->{over};
-
- return unless need_xapian($self);
- my $db = $self->{xdb};
-
- # XXX careful, we cannot use batch_do here since we conditionally
- # delete documents based on other factors, so we cannot call
- # find_doc_ids twice.
- my ($head, $tail) = $self->find_doc_ids('Q' . $mid);
- return if $head == $tail;
-
- # there is only ONE element in @delete unless we
- # have bugs in our v2writable deduplication check
- my @delete;
- for (; $head != $tail; $head++) {
- my $docid = $head->get_docid;
- my $doc = $db->get_document($docid);
- my $smsg = bless { mid => $mid }, 'PublicInbox::Smsg';
- $smsg->load_expand($doc);
- if ($smsg->{blob} eq $oid) {
- push(@delete, $docid);
- }
- }
- $db->delete_document($_) foreach @delete;
- scalar(@delete);
+ my ($self, $oid, $num) = @_;
+ die "BUG: remove_by_oid is v2-only\n" if $self->{over};
+ $self->begin_txn_lazy;
+ xdb_remove($self, $oid, $num) if need_xapian($self);
}
sub index_git_blob_id {
}
}
-sub unindex_blob {
- my ($self, $mime) = @_;
- my $mid = eval { mid_mime($mime) };
- $self->remove_message($mid) if defined $mid;
+# v1 only
+sub unindex_eml {
+ my ($self, $oid, $eml) = @_;
+ my $mids = mids($eml);
+ my $nr = 0;
+ my %tmp;
+ for my $mid (@$mids) {
+ my @removed = eval { $self->{over}->remove_oid($oid, $mid) };
+ if ($@) {
+ warn "E: failed to remove <$mid> from overview: $@\n";
+ } else {
+ $nr += scalar @removed;
+ $tmp{$_}++ for @removed;
+ }
+ }
+ if (!$nr) {
+ $mids = join('> <', @$mids);
+ warn "W: <$mids> missing for removal from overview\n";
+ }
+ while (my ($num, $nr) = each %tmp) {
+ warn "BUG: $num appears >1 times ($nr) for $oid\n" if $nr != 1;
+ }
+ xdb_remove($self, $oid, keys %tmp) if need_xapian($self);
}
sub index_mm {
$self->{mm}->mid_delete(mid_mime($mime));
}
-sub index_both {
- my ($self, $mime, $smsg) = @_;
- my $num = index_mm($self, $mime);
- $smsg->{num} = $num;
- add_message($self, $mime, $smsg);
+# returns the number of bytes to add if given a non-CRLF arg
+sub crlf_adjust ($) {
+ if (index($_[0], "\r\n") < 0) {
+ # common case is LF-only, every \n needs an \r;
+ # so favor a cheap tr// over an expensive m//g
+ $_[0] =~ tr/\n/\n/;
+ } else { # count number of '\n' w/o '\r', expensive:
+ scalar(my @n = ($_[0] =~ m/(?<!\r)\n/g));
+ }
}
-sub unindex_both {
- my ($self, $mime) = @_;
- unindex_blob($self, $mime);
- unindex_mm($self, $mime);
+sub index_both { # git->cat_async callback
+ my ($bref, $oid, $type, $size, $sync) = @_;
+ my ($nr, $max) = @$sync{qw(nr max)};
+ ++$$nr;
+ $$max -= $size;
+ $size += crlf_adjust($$bref);
+ my $smsg = bless { bytes => $size, blob => $oid }, 'PublicInbox::Smsg';
+ my $self = $sync->{sidx};
+ my $eml = PublicInbox::Eml->new($bref);
+ my $num = index_mm($self, $eml);
+ $smsg->{num} = $num;
+ add_message($self, $eml, $smsg, $sync);
}
-sub do_cat_mail {
- my ($git, $blob, $sizeref) = @_;
- my $str = $git->cat_file($blob, $sizeref) or
- die "BUG: $blob not found in $git->{git_dir}";
- PublicInbox::Eml->new($str);
+sub unindex_both { # git->cat_async callback
+ my ($bref, $oid, $type, $size, $self) = @_;
+ my $eml = PublicInbox::Eml->new($bref);
+ unindex_eml($self, $oid, $eml);
+ unindex_mm($self, $eml);
}
# called by public-inbox-index
sub index_sync {
my ($self, $opts) = @_;
delete $self->{lock_path} if $opts->{-skip_lock};
- $self->{-inbox}->with_umask(sub { $self->_index_sync($opts) })
-}
-
-sub batch_adjust ($$$$$) {
- my ($max, $bytes, $batch_cb, $latest, $nr) = @_;
- $$max -= $bytes;
- if ($$max <= 0) {
- $$max = $BATCH_BYTES;
- $batch_cb->($nr, $latest);
- }
+ $self->{ibx}->with_umask(\&_index_sync, $self, $opts);
}
-sub too_big ($$$) {
- my ($self, $git, $oid) = @_;
+sub too_big ($$) {
+ my ($self, $oid) = @_;
my $max_size = $self->{index_max_size} or return;
- my (undef, undef, $size) = $git->check($oid);
- die "E: bad $oid in $git->{git_dir}\n" if !defined($size);
+ my (undef, undef, $size) = $self->{ibx}->git->check($oid);
+ die "E: bad $oid in $self->{ibx}->{inboxdir}\n" if !defined($size);
return if $size <= $max_size;
warn "W: skipping $oid ($size > $max_size)\n";
1;
# only for v1
sub read_log {
- my ($self, $log, $add_cb, $del_cb, $batch_cb) = @_;
+ my ($self, $log, $batch_cb) = @_;
my $hex = '[a-f0-9]';
my $h40 = $hex .'{40}';
my $addmsg = qr!^:000000 100644 \S+ ($h40) A\t${hex}{2}/${hex}{38}$!;
my $delmsg = qr!^:100644 000000 ($h40) \S+ D\t${hex}{2}/${hex}{38}$!;
- my $git = $self->{git};
+ my $git = $self->{ibx}->git;
my $latest;
- my $bytes;
my $max = $BATCH_BYTES;
local $/ = "\n";
my %D;
my $line;
my $newest;
my $nr = 0;
+ my $sync = { sidx => $self, nr => \$nr, max => \$max };
while (defined($line = <$log>)) {
if ($line =~ /$addmsg/o) {
my $blob = $1;
if (delete $D{$blob}) {
+ # make sure pending index writes are done
+ # before writing to ->mm
+ $git->cat_async_wait;
+
if (defined $self->{regen_down}) {
my $num = $self->{regen_down}--;
$self->{mm}->num_highwater($num);
}
next;
}
- next if too_big($self, $git, $blob);
- my $mime = do_cat_mail($git, $blob, \$bytes);
- my $smsg = bless {}, 'PublicInbox::Smsg';
- batch_adjust(\$max, $bytes, $batch_cb, $latest, ++$nr);
- $smsg->{blob} = $blob;
- $smsg->{bytes} = $bytes;
- $add_cb->($self, $mime, $smsg);
+ next if too_big($self, $blob);
+ $git->cat_async($blob, \&index_both, { %$sync });
+ if ($max <= 0) {
+ $git->cat_async_wait;
+ $max = $BATCH_BYTES;
+ $batch_cb->($nr, $latest);
+ }
} elsif ($line =~ /$delmsg/o) {
my $blob = $1;
- $D{$blob} = 1 unless too_big($self, $git, $blob);
+ $D{$blob} = 1 unless too_big($self, $blob);
} elsif ($line =~ /^commit ($h40)/o) {
$latest = $1;
$newest ||= $latest;
} elsif ($line =~ /^author .*? ([0-9]+) [\-\+][0-9]+$/) {
- $self->{autime} = $1;
+ $sync->{autime} = $1;
} elsif ($line =~ /^committer .*? ([0-9]+) [\-\+][0-9]+$/) {
- $self->{cotime} = $1;
+ $sync->{cotime} = $1;
}
}
close($log) or die "git log failed: \$?=$?";
# get the leftovers
foreach my $blob (keys %D) {
- my $mime = do_cat_mail($git, $blob, \$bytes);
- $del_cb->($self, $mime);
+ $git->cat_async($blob, \&unindex_both, $self);
}
- delete @$self{qw(autime cotime)};
+ $git->cat_async_wait;
$batch_cb->($nr, $latest, $newest);
}
sub _git_log {
my ($self, $opts, $range) = @_;
- my $git = $self->{git};
+ my $git = $self->{ibx}->git;
if (index($range, '..') < 0) {
# don't show annoying git errors to users who run -index
} else {
# normal regen is for for fresh data
$self->{regen_down} = $fcount;
+ $self->{regen_down} += $high unless $opts->{reindex};
}
} else {
# Give oldest messages the smallest numbers
sub need_update ($$$) {
my ($self, $cur, $new) = @_;
- my $git = $self->{git};
+ my $git = $self->{ibx}->git;
return 1 if $cur && !is_ancestor($git, $cur, $new);
my $range = $cur eq '' ? $new : "$cur..$new";
chomp(my $n = $git->qx(qw(rev-list --count), $range));
$lx = $lm;
}
# Use last_commit from msgmap if it is older or unset
- if (!$lm || ($lx && $lm && is_ancestor($self->{git}, $lm, $lx))) {
+ if (!$lm || ($lx && $lm && is_ancestor($self->{ibx}->git, $lm, $lx))) {
$lx = $lm;
}
$lx;
my ($self, $opts) = @_;
my $tip = $opts->{ref} || 'HEAD';
my ($last_commit, $lx, $xlog);
- my $git = $self->{git};
+ my $git = $self->{ibx}->git;
$git->batch_prepare;
my $pr = $opts->{-progress};
my $xdb = $self->begin_txn_lazy;
+ $self->{over}->rethread_prepare($opts);
my $mm = _msgmap_init($self);
do {
- if ($xlog) {
- close($xlog) or die "git log failed: \$?=$?";
- $xlog = undef;
- }
+ $xlog = undef; # stop previous git-log via SIGPIPE
$last_commit = _last_x_commit($self, $mm);
$lx = reindex_from($opts->{reindex}, $last_commit);
} while (_last_x_commit($self, $mm) ne $last_commit);
my $dbh = $mm->{dbh} if $mm;
- my $cb = sub {
+ my $batch_cb = sub {
my ($nr, $commit, $newest) = @_;
if ($dbh) {
if ($newest) {
$xdb->set_metadata('last_commit', $newest);
}
}
+
+ $self->{over}->rethread_done($opts) if $newest; # all done
$self->commit_txn_lazy;
$git->cleanup;
- $xdb = _xdb_release($self);
- # let another process do some work... <
+ $xdb = _xdb_release($self, $nr);
+ # let another process do some work...
$pr->("indexed $nr/$self->{ntodo}\n") if $pr && $nr;
- if (!$newest) {
+ if (!$newest) { # more to come
$xdb = $self->begin_txn_lazy;
$dbh->begin_work if $dbh;
}
};
$dbh->begin_work;
- read_log($self, $xlog, *index_both, *unindex_both, $cb);
+ read_log($self, $xlog, $batch_cb);
}
sub DESTROY {
}
sub remote_remove {
- my ($self, $oid, $mid) = @_;
+ my ($self, $oid, $num) = @_;
if (my $w = $self->{w}) {
# triggers remove_by_oid in a shard
- print $w "D $oid $mid\n" or die "failed to write remove $!";
+ print $w "D $oid $num\n" or die "failed to write remove $!";
} else {
- $self->begin_txn_lazy;
- $self->remove_by_oid($oid, $mid);
+ $self->remove_by_oid($oid, $num);
}
}
-sub begin_txn_lazy {
+sub _begin_txn {
my ($self) = @_;
- return if $self->{txn};
+ my $xdb = $self->{xdb} || $self->_xdb_acquire;
+ $self->{over}->begin_lazy if $self->{over};
+ $xdb->begin_transaction if $xdb;
+ $self->{txn} = 1;
+ $xdb;
+}
- $self->{-inbox}->with_umask(sub {
- my $xdb = $self->{xdb} || $self->_xdb_acquire;
- $self->{over}->begin_lazy if $self->{over};
- $xdb->begin_transaction if $xdb;
- $self->{txn} = 1;
- $xdb;
- });
+sub begin_txn_lazy {
+ my ($self) = @_;
+ $self->{ibx}->with_umask(\&_begin_txn, $self) if !$self->{txn};
}
# store 'indexlevel=medium' in v2 shard=0 and v1 (only one shard)
}
}
+sub _commit_txn {
+ my ($self) = @_;
+ if (my $xdb = $self->{xdb}) {
+ set_indexlevel($self);
+ $xdb->commit_transaction;
+ }
+ $self->{over}->commit_lazy if $self->{over};
+}
+
sub commit_txn_lazy {
my ($self) = @_;
- delete $self->{txn} or return;
- $self->{-inbox}->with_umask(sub {
- if (my $xdb = $self->{xdb}) {
- set_indexlevel($self);
- $xdb->commit_transaction;
- }
- $self->{over}->commit_lazy if $self->{over};
- });
+ delete($self->{txn}) and
+ $self->{ibx}->with_umask(\&_commit_txn, $self);
}
sub worker_done {