use parent qw(PublicInbox::Search PublicInbox::Lock Exporter);
use PublicInbox::Eml;
use PublicInbox::InboxWritable;
-use PublicInbox::MID qw(mid_mime mids_for_index mids);
+use PublicInbox::MID qw(mids_for_index mids);
use PublicInbox::MsgIter;
use PublicInbox::IdxStack;
use Carp qw(croak);
use POSIX qw(strftime);
use PublicInbox::OverIdx;
-use PublicInbox::Spawn qw(spawn);
+use PublicInbox::Spawn qw(spawn nodatacow_dir);
use PublicInbox::Git qw(git_unquote);
use PublicInbox::MsgTime qw(msg_timestamp msg_datestamp);
-our @EXPORT_OK = qw(too_big crlf_adjust log2stack is_ancestor);
+our @EXPORT_OK = qw(crlf_adjust log2stack is_ancestor check_size);
my $X = \%PublicInbox::Search::X;
my ($DB_CREATE_OR_OPEN, $DB_OPEN);
our $DB_NO_SYNC = 0;
-our $BATCH_BYTES = defined($ENV{XAPIAN_FLUSH_THRESHOLD}) ?
- 0x7fffffff : 1_000_000;
+our $BATCH_BYTES = $ENV{XAPIAN_FLUSH_THRESHOLD} ? 0x7fffffff : 1_000_000;
use constant DEBUG => !!$ENV{DEBUG};
my $xapianlevels = qr/\A(?:full|medium)\z/;
$self->{lock_path} = "$inboxdir/ssoma.lock";
my $dir = $self->xdir;
$self->{over} = PublicInbox::OverIdx->new("$dir/over.sqlite3");
- $self->{over}->{-no_sync} = 1 if $ibx->{-no_sync};
- $self->{index_max_size} = $ibx->{index_max_size};
+ $self->{over}->{-no_fsync} = 1 if $ibx->{-no_fsync};
} elsif ($version == 2) {
defined $shard or die "shard is required for v2\n";
# shard is a number
# don't create empty Xapian directories if we don't need Xapian
my $is_shard = defined($self->{shard});
- if (!$is_shard || ($is_shard && need_xapian($self))) {
+ if (!-d $dir && (!$is_shard ||
+ ($is_shard && need_xapian($self)))) {
File::Path::mkpath($dir);
+ nodatacow_dir($dir);
}
}
return unless defined $flag;
- $flag |= $DB_NO_SYNC if $self->{ibx}->{-no_sync};
+ $flag |= $DB_NO_SYNC if $self->{ibx}->{-no_fsync};
my $xdb = eval { ($X->{WritableDatabase})->new($dir, $flag) };
if ($@) {
die "Failed opening $dir: ", $@;
}
sub add_xapian ($$$$) {
- my ($self, $mime, $smsg, $mids) = @_;
- my $hdr = $mime->header_obj;
+ my ($self, $eml, $smsg, $mids) = @_;
my $doc = $X->{Document}->new;
add_val($doc, PublicInbox::Search::TS(), $smsg->{ts});
my @ds = gmtime($smsg->{ds});
$tg->set_document($doc);
index_headers($self, $smsg);
- msg_iter($mime, \&index_xapian, [ $self, $doc ]);
- index_ids($self, $doc, $hdr, $mids);
+ msg_iter($eml, \&index_xapian, [ $self, $doc ]);
+ index_ids($self, $doc, $eml, $mids);
$smsg->{to} = $smsg->{cc} = ''; # WWW doesn't need these, only NNTP
- PublicInbox::OverIdx::parse_references($smsg, $hdr, $mids);
+ PublicInbox::OverIdx::parse_references($smsg, $eml, $mids);
my $data = $smsg->to_doc_data;
$doc->set_data($data);
if (my $altid = $self->{-altid}) {
die "BUG: _msgmap_init is only for v1\n" if $self->{ibx_ver} != 1;
$self->{mm} //= eval {
require PublicInbox::Msgmap;
- my $rw = $self->{ibx}->{-no_sync} ? 2 : 1;
+ my $rw = $self->{ibx}->{-no_fsync} ? 2 : 1;
PublicInbox::Msgmap->new($self->{ibx}->{inboxdir}, $rw);
};
}
sub add_message {
# mime = PublicInbox::Eml or Email::MIME object
my ($self, $mime, $smsg, $sync) = @_;
- my $hdr = $mime->header_obj;
- my $mids = mids_for_index($hdr);
+ my $mids = mids_for_index($mime);
$smsg //= bless { blob => '' }, 'PublicInbox::Smsg'; # test-only compat
$smsg->{mid} //= $mids->[0]; # v1 compatibility
$smsg->{num} //= do { # v1
};
# v1 and tests only:
- $smsg->populate($hdr, $sync);
+ $smsg->populate($mime, $sync);
$smsg->{bytes} //= length($mime->as_string);
eval {
while (my ($num, $nr) = each %tmp) {
warn "BUG: $num appears >1 times ($nr) for $oid\n" if $nr != 1;
}
+ if ($nr) {
+ $self->{mm}->num_delete($_) for (keys %tmp);
+ } else { # just in case msgmap and over.sqlite3 become desynched:
+ $self->{mm}->mid_delete($mids->[0]);
+ }
xdb_remove($self, $oid, keys %tmp) if need_xapian($self);
}
}
}
-sub unindex_mm {
- my ($self, $mime) = @_;
- $self->{mm}->mid_delete(mid_mime($mime));
-}
-
# returns the number of bytes to add if given a non-CRLF arg
sub crlf_adjust ($) {
if (index($_[0], "\r\n") < 0) {
sub unindex_both { # git->cat_async callback
my ($bref, $oid, $type, $size, $self) = @_;
- my $eml = PublicInbox::Eml->new($bref);
- unindex_eml($self, $oid, $eml);
- unindex_mm($self, $eml);
+ unindex_eml($self, $oid, PublicInbox::Eml->new($bref));
}
# called by public-inbox-index
}
}
-sub too_big ($$) {
- my ($self, $oid) = @_;
- my $max_size = $self->{index_max_size} or return;
- my (undef, undef, $size) = $self->{ibx}->git->check($oid);
- die "E: bad $oid in $self->{ibx}->{inboxdir}\n" if !defined($size);
- return if $size <= $max_size;
- warn "W: skipping $oid ($size > $max_size)\n";
- 1;
-}
-
-sub ck_size { # check_async cb for -index --max-size=...
+sub check_size { # check_async cb for -index --max-size=...
my ($oid, $type, $size, $arg, $git) = @_;
(($type // '') eq 'blob') or die "E: bad $oid in $git->{git_dir}";
- if ($size <= $arg->{index_max_size}) {
- $git->cat_async($oid, \&index_both, $arg);
+ if ($size <= $arg->{max_size}) {
+ $git->cat_async($oid, $arg->{index_oid}, $arg);
} else {
- warn "W: skipping $oid ($size > $arg->{index_max_size})\n";
+ warn "W: skipping $oid ($size > $arg->{max_size})\n";
}
}
$self->{mm}->last_commit($newest);
}
} else {
- ${$sync->{max}} = $BATCH_BYTES;
+ ${$sync->{max}} = $self->{batch_bytes};
}
$self->{mm}->{dbh}->commit;
sub process_stack {
my ($self, $sync, $stk) = @_;
my $git = $self->{ibx}->git;
- my $max = $BATCH_BYTES;
+ my $max = $self->{batch_bytes};
my $nr = 0;
$sync->{nr} = \$nr;
$sync->{max} = \$max;
$git->cat_async($oid, \&unindex_both, $self);
}
}
- $sync->{index_max_size} = $self->{ibx}->{index_max_size};
+ if ($sync->{max_size} = $sync->{-opt}->{max_size}) {
+ $sync->{index_oid} = \&index_both;
+ }
while (my ($f, $at, $ct, $oid) = $stk->pop_rec) {
if ($f eq 'm') {
my $arg = { %$sync, autime => $at, cotime => $ct };
- if ($sync->{index_max_size}) {
- $git->check_async($oid, \&ck_size, $arg);
+ if ($sync->{max_size}) {
+ $git->check_async($oid, \&check_size, $arg);
} else {
$git->cat_async($oid, \&index_both, $arg);
}
my ($self, $opts) = @_;
my $tip = $opts->{ref} || 'HEAD';
my $git = $self->{ibx}->git;
+ $self->{batch_bytes} = $opts->{batch_size} // $BATCH_BYTES;
$git->batch_prepare;
my $pr = $opts->{-progress};
my $sync = { reindex => $opts->{reindex}, -opt => $opts };