X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FSearchIdx.pm;h=6e2e614c5085c7e45fa33ad06375a9137af1ee31;hb=7ffa98cbbf5bff8225fdfedb0f2b7c2beb5d0a6f;hp=826302dec1270ca498e455239beabe39260b9ba5;hpb=7355803ccf17bd609c9d667d3b9cadb2adfc31bd;p=public-inbox.git diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm index 826302de..6e2e614c 100644 --- a/lib/PublicInbox/SearchIdx.pm +++ b/lib/PublicInbox/SearchIdx.pm @@ -11,6 +11,7 @@ use strict; use v5.10.1; use parent qw(PublicInbox::Search PublicInbox::Lock Exporter); use PublicInbox::Eml; +use PublicInbox::Search qw(xap_terms); use PublicInbox::InboxWritable; use PublicInbox::MID qw(mids_for_index mids); use PublicInbox::MsgIter; @@ -22,17 +23,24 @@ use PublicInbox::OverIdx; use PublicInbox::Spawn qw(spawn nodatacow_dir); use PublicInbox::Git qw(git_unquote); use PublicInbox::MsgTime qw(msg_timestamp msg_datestamp); +use PublicInbox::Address; +use Config; our @EXPORT_OK = qw(log2stack is_ancestor check_size prepare_stack index_text term_generator add_val is_bad_blob); my $X = \%PublicInbox::Search::X; our ($DB_CREATE_OR_OPEN, $DB_OPEN); our $DB_NO_SYNC = 0; -our $BATCH_BYTES = $ENV{XAPIAN_FLUSH_THRESHOLD} ? 0x7fffffff : 1_000_000; +our $BATCH_BYTES = $ENV{XAPIAN_FLUSH_THRESHOLD} ? 0x7fffffff : + # assume a typical 64-bit system has 8x more RAM than a + # typical 32-bit system: + (($Config{ptrsize} >= 8 ? 8192 : 1024) * 1024); + use constant DEBUG => !!$ENV{DEBUG}; my $xapianlevels = qr/\A(?:full|medium)\z/; my $hex = '[a-f0-9]'; my $OID = $hex .'{40,}'; +my @VMD_MAP = (kw => 'K', L => 'L'); our $INDEXLEVELS = qr/\A(?:full|medium|basic)\z/; sub new { @@ -86,7 +94,7 @@ sub need_xapian ($) { $_[0]->{indexlevel} =~ $xapianlevels } sub idx_release { my ($self, $wake) = @_; if (need_xapian($self)) { - my $xdb = delete $self->{xdb} or croak 'not acquired'; + my $xdb = delete $self->{xdb} or croak '{xdb} not acquired'; $xdb->close; } $self->lock_release($wake) if $self->{creat}; @@ -95,7 +103,7 @@ sub idx_release { sub load_xapian_writable () { return 1 if $X->{WritableDatabase}; - PublicInbox::Search::load_xapian() or return; + PublicInbox::Search::load_xapian() or die "failed to load Xapian: $@\n"; my $xap = $PublicInbox::Search::Xap; for (qw(Document TermGenerator WritableDatabase)) { $X->{$_} = $xap.'::'.$_; @@ -158,22 +166,44 @@ sub term_generator ($) { # write-only } } +sub index_phrase ($$$$) { + my ($self, $text, $wdf_inc, $prefix) = @_; + + my $tg = term_generator($self); + $tg->index_text($text, $wdf_inc, $prefix); + $tg->increase_termpos; +} + sub index_text ($$$$) { my ($self, $text, $wdf_inc, $prefix) = @_; - my $tg = term_generator($self); # man Search::Xapian::TermGenerator if ($self->{indexlevel} eq 'full') { - $tg->index_text($text, $wdf_inc, $prefix); - $tg->increase_termpos; + index_phrase($self, $text, $wdf_inc, $prefix); } else { + my $tg = term_generator($self); $tg->index_text_without_positions($text, $wdf_inc, $prefix); } } sub index_headers ($$) { my ($self, $smsg) = @_; - my @x = (from => 'A', # Author - subject => 'S', to => 'XTO', cc => 'XCC'); + my @x = (from => 'A', to => 'XTO', cc => 'XCC'); # A: Author + while (my ($field, $pfx) = splice(@x, 0, 2)) { + my $val = $smsg->{$field}; + next if $val eq ''; + # include "(comments)" after the address, too, so not using + # PublicInbox::Address::names or pairs + index_text($self, $val, 1, $pfx); + + # we need positional info for email addresses since they + # can be considered phrases + if ($self->{indexlevel} eq 'medium') { + for my $addr (PublicInbox::Address::emails($val)) { + index_phrase($self, $addr, 1, $pfx); + } + } + } + @x = (subject => 'S'); while (my ($field, $pfx) = splice(@x, 0, 2)) { my $val = $smsg->{$field}; index_text($self, $val, 1, $pfx) if $val ne ''; @@ -186,7 +216,11 @@ sub index_diff_inc ($$$$) { index_text($self, join("\n", @$xnq), 1, 'XNQ'); @$xnq = (); } - index_text($self, $text, 1, $pfx); + if ($pfx eq 'XDFN') { + index_phrase($self, $text, 1, $pfx); + } else { + index_text($self, $text, 1, $pfx); + } } sub index_old_diff_fn { @@ -225,6 +259,7 @@ sub index_diff ($$$) { } elsif (m!^diff --git "?[^/]+/.+ "?[^/]+/.+\z!) { # wait until "---" and "+++" to capture filenames $in_diff = 1; + push @xnq, $_; # traditional diff: } elsif (m/^diff -(.+) (\S+) (\S+)$/) { my ($opt, $fa, $fb) = ($1, $2, $3); @@ -292,7 +327,7 @@ sub index_xapian { # msg_iter callback my $ct = $part->content_type || 'text/plain'; my $fn = $part->filename; if (defined $fn && $fn ne '') { - index_text($self, $fn, 1, 'XFN'); + index_phrase($self, $fn, 1, 'XFN'); } if ($part->{is_submsg}) { my $mids = mids_for_index($part); @@ -330,20 +365,20 @@ sub index_list_id ($$$) { $l =~ /<([^>]+)>/ or next; my $lid = lc $1; $doc->add_boolean_term('G' . $lid); - index_text($self, $lid, 1, 'XL'); # probabilistic + index_phrase($self, $lid, 1, 'XL'); # probabilistic } } sub index_ids ($$$$) { my ($self, $doc, $hdr, $mids) = @_; for my $mid (@$mids) { - index_text($self, $mid, 1, 'XM'); + index_phrase($self, $mid, 1, 'XM'); # because too many Message-IDs are prefixed with # "Pine.LNX."... if ($mid =~ /\w{12,}/) { my @long = ($mid =~ /(\w{3,}+)/g); - index_text($self, join(' ', @long), 1, 'XM'); + index_phrase($self, join(' ', @long), 1, 'XM'); } } $doc->add_boolean_term('Q' . $_) for @$mids; @@ -401,17 +436,25 @@ sub eml2doc ($$$;$) { sub add_xapian ($$$$) { my ($self, $eml, $smsg, $mids) = @_; begin_txn_lazy($self); + my $merge_vmd = delete $smsg->{-merge_vmd}; my $doc = eml2doc($self, $eml, $smsg, $mids); + if (my $old = $merge_vmd ? _get_doc($self, $smsg->{num}) : undef) { + my @x = @VMD_MAP; + while (my ($field, $pfx) = splice(@x, 0, 2)) { + for my $term (xap_terms($pfx, $old)) { + $doc->add_boolean_term($pfx.$term); + } + } + } $self->{xdb}->replace_document($smsg->{num}, $doc); } sub _msgmap_init ($) { my ($self) = @_; die "BUG: _msgmap_init is only for v1\n" if $self->{ibx}->version != 1; - $self->{mm} //= eval { + $self->{mm} //= do { require PublicInbox::Msgmap; - my $rw = $self->{ibx}->{-no_fsync} ? 2 : 1; - PublicInbox::Msgmap->new($self->{ibx}->{inboxdir}, $rw); + PublicInbox::Msgmap->new_file($self->{ibx}, 1); }; } @@ -467,11 +510,20 @@ sub add_eidx_info { begin_txn_lazy($self); my $doc = _get_doc($self, $docid) or return; term_generator($self)->set_document($doc); + + # '.' is special for lei_store $doc->add_boolean_term('O'.$eidx_key) if $eidx_key ne '.'; + index_list_id($self, $doc, $eml); $self->{xdb}->replace_document($docid, $doc); } +sub get_terms { + my ($self, $pfx, $docid) = @_; + begin_txn_lazy($self); + xap_terms($pfx, $self->{xdb}, $docid); +} + sub remove_eidx_info { my ($self, $docid, $eidx_key, $eml) = @_; begin_txn_lazy($self); @@ -501,69 +553,114 @@ sub remove_eidx_info { $self->{xdb}->replace_document($docid, $doc); } -sub set_keywords { - my ($self, $docid, @kw) = @_; +sub set_vmd { + my ($self, $docid, $vmd) = @_; begin_txn_lazy($self); my $doc = _get_doc($self, $docid) or return; - my %keep = map { $_ => 1 } @kw; - my %add = %keep; - my @rm; - my $end = $doc->termlist_end; - for (my $cur = $doc->termlist_begin; $cur != $end; $cur++) { - $cur->skip_to('K'); - last if $cur == $end; - my $kw = $cur->get_termname; - $kw =~ s/\AK//s or next; - $keep{$kw} ? delete($add{$kw}) : push(@rm, $kw); - } - return unless (scalar(@rm) + scalar(keys %add)); - $doc->remove_term('K'.$_) for @rm; - $doc->add_boolean_term('K'.$_) for (keys %add); + my ($end, @rm, @add); + my @x = @VMD_MAP; + while (my ($field, $pfx) = splice(@x, 0, 2)) { + my $set = $vmd->{$field} // next; + my %keep = map { $_ => 1 } @$set; + my %add = %keep; + $end //= $doc->termlist_end; + for (my $cur = $doc->termlist_begin; $cur != $end; $cur++) { + $cur->skip_to($pfx); + last if $cur == $end; + my $v = $cur->get_termname; + $v =~ s/\A$pfx//s or next; + $keep{$v} ? delete($add{$v}) : push(@rm, $pfx.$v); + } + push(@add, map { $pfx.$_ } keys %add); + } + return unless scalar(@rm) || scalar(@add); + $doc->remove_term($_) for @rm; + $doc->add_boolean_term($_) for @add; $self->{xdb}->replace_document($docid, $doc); } -sub add_keywords { - my ($self, $docid, @kw) = @_; +sub apply_vmd_mod ($$) { + my ($doc, $vmd_mod) = @_; + my $updated = 0; + my @x = @VMD_MAP; + while (my ($field, $pfx) = splice(@x, 0, 2)) { + # field: "L" or "kw" + for my $val (@{$vmd_mod->{"-$field"} // []}) { + eval { + $doc->remove_term($pfx . $val); + ++$updated; + }; + } + for my $val (@{$vmd_mod->{"+$field"} // []}) { + $doc->add_boolean_term($pfx . $val); + ++$updated; + } + } + $updated; +} + +sub add_vmd { + my ($self, $docid, $vmd) = @_; begin_txn_lazy($self); my $doc = _get_doc($self, $docid) or return; - $doc->add_boolean_term('K'.$_) for @kw; - $self->{xdb}->replace_document($docid, $doc); + my @x = @VMD_MAP; + my $updated = 0; + while (my ($field, $pfx) = splice(@x, 0, 2)) { + my $add = $vmd->{$field} // next; + $doc->add_boolean_term($pfx . $_) for @$add; + $updated += scalar(@$add); + } + $updated += apply_vmd_mod($doc, $vmd); + $self->{xdb}->replace_document($docid, $doc) if $updated; } -sub remove_keywords { - my ($self, $docid, @kw) = @_; +sub remove_vmd { + my ($self, $docid, $vmd) = @_; begin_txn_lazy($self); my $doc = _get_doc($self, $docid) or return; my $replace; - eval { - $doc->remove_term('K'.$_); - $replace = 1 - } for @kw; + my @x = @VMD_MAP; + while (my ($field, $pfx) = splice(@x, 0, 2)) { + my $rm = $vmd->{$field} // next; + for (@$rm) { + eval { + $doc->remove_term($pfx . $_); + $replace = 1; + }; + } + } $self->{xdb}->replace_document($docid, $doc) if $replace; } -sub smsg_from_doc ($) { - my ($doc) = @_; - my $data = $doc->get_data or return; - my $smsg = bless {}, 'PublicInbox::Smsg'; - $smsg->{ts} = int_val($doc, PublicInbox::Search::TS()); - my $dt = int_val($doc, PublicInbox::Search::DT()); - my ($yyyy, $mon, $dd, $hh, $mm, $ss) = unpack('A4A2A2A2A2A2', $dt); - $smsg->{ds} = timegm($ss, $mm, $hh, $dd, $mon - 1, $yyyy); - $smsg->load_from_data($data); - $smsg; +sub update_vmd { + my ($self, $docid, $vmd_mod) = @_; + begin_txn_lazy($self); + my $doc = _get_doc($self, $docid) or return; + my $updated = apply_vmd_mod($doc, $vmd_mod); + $self->{xdb}->replace_document($docid, $doc) if $updated; + $updated; } sub xdb_remove { my ($self, @docids) = @_; - $self->begin_txn_lazy; - my $xdb = $self->{xdb} or return; + begin_txn_lazy($self); + my $xdb = $self->{xdb} // die 'BUG: missing {xdb}'; for my $docid (@docids) { eval { $xdb->delete_document($docid) }; warn "E: #$docid not in in Xapian? $@\n" if $@; } } +sub xdb_remove_quiet { + my ($self, $docid) = @_; + begin_txn_lazy($self); + my $xdb = $self->{xdb} // die 'BUG: missing {xdb}'; + eval { $xdb->delete_document($docid) }; + ++$self->{-quiet_rm} unless $@; +} + +sub nr_quiet_rm { delete($_[0]->{-quiet_rm}) // 0 } + sub index_git_blob_id { my ($doc, $pfx, $objid) = @_; @@ -669,7 +766,8 @@ sub index_sync { my ($self, $opt) = @_; delete $self->{lock_path} if $opt->{-skip_lock}; $self->with_umask(\&_index_sync, $self, $opt); - if ($opt->{reindex} && !$opt->{quit}) { + if ($opt->{reindex} && !$opt->{quit} && + !grep(defined, @$opt{qw(since until)})) { my %again = %$opt; delete @again{qw(rethread reindex)}; index_sync($self, \%again); @@ -694,18 +792,19 @@ sub v1_checkpoint ($$;$) { # $newest may be undef my $newest = $stk ? $stk->{latest_cmt} : ${$sync->{latest_cmt}}; if (defined($newest)) { - my $cur = $self->{mm}->last_commit || ''; - if (need_update($self, $cur, $newest)) { + my $cur = $self->{mm}->last_commit; + if (need_update($self, $sync, $cur, $newest)) { $self->{mm}->last_commit($newest); } } ${$sync->{max}} = $self->{batch_bytes}; $self->{mm}->{dbh}->commit; - my $xdb = need_xapian($self) ? $self->{xdb} : undef; + eval { $self->{mm}->{dbh}->do('PRAGMA optimize') }; + my $xdb = $self->{xdb}; if ($newest && $xdb) { my $cur = $xdb->get_metadata('last_commit'); - if (need_update($self, $cur, $newest)) { + if (need_update($self, $sync, $cur, $newest)) { $xdb->set_metadata('last_commit', $newest); } } @@ -789,9 +888,14 @@ sub log2stack ($$$) { # Count the new files so they can be added newest to oldest # and still have numbers increasing from oldest to newest - my $fh = $git->popen(qw(log --raw -r --pretty=tformat:%at-%ct-%H - --no-notes --no-color --no-renames --no-abbrev), - $range); + my @cmd = qw(log --raw -r --pretty=tformat:%at-%ct-%H + --no-notes --no-color --no-renames --no-abbrev); + for my $k (qw(since until)) { + my $v = $sync->{-opt}->{$k} // next; + next if !$sync->{-opt}->{reindex}; + push @cmd, "--$k=$v"; + } + my $fh = $git->popen(@cmd, $range); my ($at, $ct, $stk, $cmt); while (<$fh>) { return if $sync->{quit}; @@ -847,10 +951,17 @@ sub is_ancestor ($$$) { $? == 0; } -sub need_update ($$$) { - my ($self, $cur, $new) = @_; +sub need_update ($$$$) { + my ($self, $sync, $cur, $new) = @_; my $git = $self->{ibx}->git; - return 1 if $cur && !is_ancestor($git, $cur, $new); + $cur //= ''; # XS Search::Xapian ->get_metadata doesn't give undef + + # don't rewind if --{since,until,before,after} are in use + return if $cur ne '' && + grep(defined, @{$sync->{-opt}}{qw(since until)}) && + is_ancestor($git, $new, $cur); + + return 1 if $cur ne '' && !is_ancestor($git, $cur, $new); my $range = $cur eq '' ? $new : "$cur..$new"; chomp(my $n = $git->qx(qw(rev-list --count), $range)); ($n eq '' || $n > 0); @@ -1004,25 +1115,4 @@ sub eidx_shard_new { $self; } -# ensure there's no stale Xapian docs by treating $over as canonical -sub over_check { - my ($self, $over) = @_; - begin_txn_lazy($self); - my $sth = $over->dbh->prepare(<<''); -SELECT COUNT(*) FROM over WHERE num = ? - - my $xdb = $self->{xdb}; - my $cur = $xdb->postlist_begin(''); - my $end = $xdb->postlist_end(''); - my $xdir = $self->xdir; - for (; $cur != $end; $cur++) { - my $docid = $cur->get_docid; - $sth->execute($docid); - my $x = $sth->fetchrow_array; - next if $x > 0; - warn "I: removing $xdir #$docid, not in `over'\n"; - $xdb->delete_document($docid); - } -} - 1;