use PublicInbox::Git qw(git_unquote);
use PublicInbox::MsgTime qw(msg_timestamp msg_datestamp);
use PublicInbox::Address;
+use Config;
our @EXPORT_OK = qw(log2stack is_ancestor check_size prepare_stack
index_text term_generator add_val is_bad_blob);
my $X = \%PublicInbox::Search::X;
our ($DB_CREATE_OR_OPEN, $DB_OPEN);
our $DB_NO_SYNC = 0;
-our $BATCH_BYTES = $ENV{XAPIAN_FLUSH_THRESHOLD} ? 0x7fffffff : 1_000_000;
+our $BATCH_BYTES = $ENV{XAPIAN_FLUSH_THRESHOLD} ? 0x7fffffff :
+ # assume a typical 64-bit system has 8x more RAM than a
+ # typical 32-bit system:
+ (($Config{ptrsize} >= 8 ? 8192 : 1024) * 1024);
+
use constant DEBUG => !!$ENV{DEBUG};
my $xapianlevels = qr/\A(?:full|medium)\z/;
sub idx_release {
my ($self, $wake) = @_;
if (need_xapian($self)) {
- my $xdb = delete $self->{xdb} or croak 'not acquired';
+ my $xdb = delete $self->{xdb} or croak '{xdb} not acquired';
$xdb->close;
}
$self->lock_release($wake) if $self->{creat};
sub load_xapian_writable () {
return 1 if $X->{WritableDatabase};
- PublicInbox::Search::load_xapian() or return;
+ PublicInbox::Search::load_xapian() or die "failed to load Xapian: $@\n";
my $xap = $PublicInbox::Search::Xap;
for (qw(Document TermGenerator WritableDatabase)) {
$X->{$_} = $xap.'::'.$_;
if (my $old = $merge_vmd ? _get_doc($self, $smsg->{num}) : undef) {
my @x = @VMD_MAP;
while (my ($field, $pfx) = splice(@x, 0, 2)) {
- my $vals = xap_terms($pfx, $old);
- $doc->add_boolean_term($pfx.$_) for keys %$vals;
+ for my $term (xap_terms($pfx, $old)) {
+ $doc->add_boolean_term($pfx.$term);
+ }
}
}
$self->{xdb}->replace_document($smsg->{num}, $doc);
$self->{xdb}->replace_document($docid, $doc);
}
+sub apply_vmd_mod ($$) {
+ my ($doc, $vmd_mod) = @_;
+ my $updated = 0;
+ my @x = @VMD_MAP;
+ while (my ($field, $pfx) = splice(@x, 0, 2)) {
+ # field: "L" or "kw"
+ for my $val (@{$vmd_mod->{"-$field"} // []}) {
+ eval {
+ $doc->remove_term($pfx . $val);
+ ++$updated;
+ };
+ }
+ for my $val (@{$vmd_mod->{"+$field"} // []}) {
+ $doc->add_boolean_term($pfx . $val);
+ ++$updated;
+ }
+ }
+ $updated;
+}
+
sub add_vmd {
my ($self, $docid, $vmd) = @_;
begin_txn_lazy($self);
my $doc = _get_doc($self, $docid) or return;
my @x = @VMD_MAP;
+ my $updated = 0;
while (my ($field, $pfx) = splice(@x, 0, 2)) {
my $add = $vmd->{$field} // next;
$doc->add_boolean_term($pfx . $_) for @$add;
+ $updated += scalar(@$add);
}
- $self->{xdb}->replace_document($docid, $doc);
+ $updated += apply_vmd_mod($doc, $vmd);
+ $self->{xdb}->replace_document($docid, $doc) if $updated;
}
sub remove_vmd {
my ($self, $docid, $vmd_mod) = @_;
begin_txn_lazy($self);
my $doc = _get_doc($self, $docid) or return;
- my $updated = 0;
- my @x = @VMD_MAP;
- while (my ($field, $pfx) = splice(@x, 0, 2)) {
- # field: "label" or "kw"
- for my $val (@{$vmd_mod->{"-$field"} // []}) {
- eval {
- $doc->remove_term($pfx . $val);
- ++$updated;
- };
- }
- for my $val (@{$vmd_mod->{"+$field"} // []}) {
- $doc->add_boolean_term($pfx . $val);
- ++$updated;
- }
- }
+ my $updated = apply_vmd_mod($doc, $vmd_mod);
$self->{xdb}->replace_document($docid, $doc) if $updated;
$updated;
}
${$sync->{max}} = $self->{batch_bytes};
$self->{mm}->{dbh}->commit;
- my $xdb = need_xapian($self) ? $self->{xdb} : undef;
+ my $xdb = $self->{xdb};
if ($newest && $xdb) {
my $cur = $xdb->get_metadata('last_commit');
if (need_update($self, $cur, $newest)) {