X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FSearchIdx.pm;h=53ec23a5c50c4facfcf1977ea04f2fa4498d8805;hb=d0079a334fe2b769e4f81d6bd3c7e4346a8fa2b2;hp=f4f3ba34f5c951f3a0b3045a7babbb3cbd0f0934;hpb=a23c5b9673a374b6abf4416824796cec5446dd9a;p=public-inbox.git diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm index f4f3ba34..53ec23a5 100644 --- a/lib/PublicInbox/SearchIdx.pm +++ b/lib/PublicInbox/SearchIdx.pm @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2021 all contributors +# Copyright (C) all contributors # License: AGPL-3.0+ # based on notmuch, but with no concept of folders, files # @@ -18,9 +18,10 @@ use PublicInbox::MsgIter; use PublicInbox::IdxStack; use Carp qw(croak carp); use POSIX qw(strftime); +use Fcntl qw(SEEK_SET); use Time::Local qw(timegm); use PublicInbox::OverIdx; -use PublicInbox::Spawn qw(spawn nodatacow_dir); +use PublicInbox::Spawn qw(spawn); use PublicInbox::Git qw(git_unquote); use PublicInbox::MsgTime qw(msg_timestamp msg_datestamp); use PublicInbox::Address; @@ -30,6 +31,7 @@ our @EXPORT_OK = qw(log2stack is_ancestor check_size prepare_stack my $X = \%PublicInbox::Search::X; our ($DB_CREATE_OR_OPEN, $DB_OPEN); our $DB_NO_SYNC = 0; +our $DB_DANGEROUS = 0; our $BATCH_BYTES = $ENV{XAPIAN_FLUSH_THRESHOLD} ? 0x7fffffff : # assume a typical 64-bit system has 8x more RAM than a # typical 32-bit system: @@ -94,7 +96,7 @@ sub need_xapian ($) { $_[0]->{indexlevel} =~ $xapianlevels } sub idx_release { my ($self, $wake) = @_; if (need_xapian($self)) { - my $xdb = delete $self->{xdb} or croak 'not acquired'; + my $xdb = delete $self->{xdb} or croak '{xdb} not acquired'; $xdb->close; } $self->lock_release($wake) if $self->{creat}; @@ -103,7 +105,7 @@ sub idx_release { sub load_xapian_writable () { return 1 if $X->{WritableDatabase}; - PublicInbox::Search::load_xapian() or return; + PublicInbox::Search::load_xapian() or die "failed to load Xapian: $@\n"; my $xap = $PublicInbox::Search::Xap; for (qw(Document TermGenerator WritableDatabase)) { $X->{$_} = $xap.'::'.$_; @@ -115,7 +117,10 @@ sub load_xapian_writable () { my $ver = (eval($xap.'::major_version()') << 16) | (eval($xap.'::minor_version()') << 8) | eval($xap.'::revision()'); - $DB_NO_SYNC = 0x4 if $ver >= 0x10400; + if ($ver >= 0x10400) { + $DB_NO_SYNC = 0x4; + $DB_DANGEROUS = 0x10; + } # Xapian v1.2.21..v1.2.24 were missing close-on-exec on OFD locks $X->{CLOEXEC_UNSET} = 1 if $ver >= 0x010215 && $ver <= 0x010218; 1; @@ -139,8 +144,12 @@ sub idx_acquire { if (!-d $dir && (!$is_shard || ($is_shard && need_xapian($self)))) { File::Path::mkpath($dir); - nodatacow_dir($dir); + require PublicInbox::Syscall; + PublicInbox::Syscall::nodatacow_dir($dir); $self->{-set_has_threadid_once} = 1; + if (($self->{ibx} // $self->{eidx})->{-dangerous}) { + $flag |= $DB_DANGEROUS; + } } } return unless defined $flag; @@ -228,8 +237,8 @@ sub index_old_diff_fn { # no renames or space support for traditional diffs, # find the number of leading common paths to strip: - my @fa = split('/', $fa); - my @fb = split('/', $fb); + my @fa = split(m'/', $fa); + my @fb = split(m'/', $fb); while (scalar(@fa) && scalar(@fb)) { $fa = join('/', @fa); $fb = join('/', @fb); @@ -259,6 +268,7 @@ sub index_diff ($$$) { } elsif (m!^diff --git "?[^/]+/.+ "?[^/]+/.+\z!) { # wait until "---" and "+++" to capture filenames $in_diff = 1; + push @xnq, $_; # traditional diff: } elsif (m/^diff -(.+) (\S+) (\S+)$/) { my ($opt, $fa, $fb) = ($1, $2, $3); @@ -269,12 +279,12 @@ sub index_diff ($$$) { $xnq); } elsif (m!^--- ("?[^/]+/.+)!) { my $fn = $1; - $fn = (split('/', git_unquote($fn), 2))[1]; + $fn = (split(m'/', git_unquote($fn), 2))[1]; $seen{$fn}++ or index_diff_inc($self, $fn, 'XDFN', $xnq); $in_diff = 1; } elsif (m!^\+\+\+ ("?[^/]+/.+)!) { my $fn = $1; - $fn = (split('/', git_unquote($fn), 2))[1]; + $fn = (split(m'/', git_unquote($fn), 2))[1]; $seen{$fn}++ or index_diff_inc($self, $fn, 'XDFN', $xnq); $in_diff = 1; } elsif (/^--- (\S+)/) { @@ -340,6 +350,20 @@ sub index_xapian { # msg_iter callback defined $s or return; $_[0]->[0] = $part = undef; # free memory + if ($s =~ /^(?:diff|---|\+\+\+) /ms) { + open(my $fh, '+>:utf8', undef) or die "open: $!"; + open(my $eh, '+>', undef) or die "open: $!"; + $fh->autoflush(1); + print $fh $s or die "print: $!"; + sysseek($fh, 0, SEEK_SET) or die "sysseek: $!"; + my $id = ($self->{ibx} // $self->{eidx})->git->qx( + [qw(patch-id --stable)], + {}, { 0 => $fh, 2 => $eh }); + $id =~ /\A([a-f0-9]{40,})/ and $doc->add_term('XDFID'.$1); + seek($eh, 0, SEEK_SET) or die "seek: $!"; + while (<$eh>) { warn $_ } + } + # split off quoted and unquoted blocks: my @sections = PublicInbox::MsgIter::split_quotes($s); undef $s; # free memory @@ -451,10 +475,9 @@ sub add_xapian ($$$$) { sub _msgmap_init ($) { my ($self) = @_; die "BUG: _msgmap_init is only for v1\n" if $self->{ibx}->version != 1; - $self->{mm} //= eval { + $self->{mm} //= do { require PublicInbox::Msgmap; - my $rw = $self->{ibx}->{-no_fsync} ? 2 : 1; - PublicInbox::Msgmap->new($self->{ibx}->{inboxdir}, $rw); + PublicInbox::Msgmap->new_file($self->{ibx}, 1); }; } @@ -518,6 +541,12 @@ sub add_eidx_info { $self->{xdb}->replace_document($docid, $doc); } +sub get_terms { + my ($self, $pfx, $docid) = @_; + begin_txn_lazy($self); + xap_terms($pfx, $self->{xdb}, $docid); +} + sub remove_eidx_info { my ($self, $docid, $eidx_key, $eml) = @_; begin_txn_lazy($self); @@ -637,14 +666,24 @@ sub update_vmd { sub xdb_remove { my ($self, @docids) = @_; - $self->begin_txn_lazy; - my $xdb = $self->{xdb} or return; + begin_txn_lazy($self); + my $xdb = $self->{xdb} // die 'BUG: missing {xdb}'; for my $docid (@docids) { eval { $xdb->delete_document($docid) }; warn "E: #$docid not in in Xapian? $@\n" if $@; } } +sub xdb_remove_quiet { + my ($self, $docid) = @_; + begin_txn_lazy($self); + my $xdb = $self->{xdb} // die 'BUG: missing {xdb}'; + eval { $xdb->delete_document($docid) }; + ++$self->{-quiet_rm} unless $@; +} + +sub nr_quiet_rm { delete($_[0]->{-quiet_rm}) // 0 } + sub index_git_blob_id { my ($doc, $pfx, $objid) = @_; @@ -750,7 +789,8 @@ sub index_sync { my ($self, $opt) = @_; delete $self->{lock_path} if $opt->{-skip_lock}; $self->with_umask(\&_index_sync, $self, $opt); - if ($opt->{reindex} && !$opt->{quit}) { + if ($opt->{reindex} && !$opt->{quit} && + !grep(defined, @$opt{qw(since until)})) { my %again = %$opt; delete @again{qw(rethread reindex)}; index_sync($self, \%again); @@ -775,18 +815,19 @@ sub v1_checkpoint ($$;$) { # $newest may be undef my $newest = $stk ? $stk->{latest_cmt} : ${$sync->{latest_cmt}}; if (defined($newest)) { - my $cur = $self->{mm}->last_commit || ''; - if (need_update($self, $cur, $newest)) { + my $cur = $self->{mm}->last_commit; + if (need_update($self, $sync, $cur, $newest)) { $self->{mm}->last_commit($newest); } } ${$sync->{max}} = $self->{batch_bytes}; $self->{mm}->{dbh}->commit; - my $xdb = need_xapian($self) ? $self->{xdb} : undef; + eval { $self->{mm}->{dbh}->do('PRAGMA optimize') }; + my $xdb = $self->{xdb}; if ($newest && $xdb) { my $cur = $xdb->get_metadata('last_commit'); - if (need_update($self, $cur, $newest)) { + if (need_update($self, $sync, $cur, $newest)) { $xdb->set_metadata('last_commit', $newest); } } @@ -870,23 +911,28 @@ sub log2stack ($$$) { # Count the new files so they can be added newest to oldest # and still have numbers increasing from oldest to newest - my $fh = $git->popen(qw(log --raw -r --pretty=tformat:%at-%ct-%H - --no-notes --no-color --no-renames --no-abbrev), - $range); - my ($at, $ct, $stk, $cmt); - while (<$fh>) { + my @cmd = qw(log --raw -r --pretty=tformat:%at-%ct-%H + --no-notes --no-color --no-renames --no-abbrev); + for my $k (qw(since until)) { + my $v = $sync->{-opt}->{$k} // next; + next if !$sync->{-opt}->{reindex}; + push @cmd, "--$k=$v"; + } + my $fh = $git->popen(@cmd, $range); + my ($at, $ct, $stk, $cmt, $l); + while (defined($l = <$fh>)) { return if $sync->{quit}; - if (/\A([0-9]+)-([0-9]+)-($OID)$/o) { + if ($l =~ /\A([0-9]+)-([0-9]+)-($OID)$/o) { ($at, $ct, $cmt) = ($1 + 0, $2 + 0, $3); $stk //= PublicInbox::IdxStack->new($cmt); - } elsif (/$del/) { + } elsif ($l =~ /$del/) { my $oid = $1; if ($D) { # reindex case $D->{pack('H*', $oid)}++; } else { # non-reindex case: $stk->push_rec('d', $at, $ct, $oid, $cmt); } - } elsif (/$add/) { + } elsif ($l =~ /$add/) { my $oid = $1; if ($D) { my $oid_bin = pack('H*', $oid); @@ -928,10 +974,17 @@ sub is_ancestor ($$$) { $? == 0; } -sub need_update ($$$) { - my ($self, $cur, $new) = @_; +sub need_update ($$$$) { + my ($self, $sync, $cur, $new) = @_; my $git = $self->{ibx}->git; - return 1 if $cur && !is_ancestor($git, $cur, $new); + $cur //= ''; # XS Search::Xapian ->get_metadata doesn't give undef + + # don't rewind if --{since,until,before,after} are in use + return if $cur ne '' && + grep(defined, @{$sync->{-opt}}{qw(since until)}) && + is_ancestor($git, $new, $cur); + + return 1 if $cur ne '' && !is_ancestor($git, $cur, $new); my $range = $cur eq '' ? $new : "$cur..$new"; chomp(my $n = $git->qx(qw(rev-list --count), $range)); ($n eq '' || $n > 0); @@ -1085,25 +1138,4 @@ sub eidx_shard_new { $self; } -# ensure there's no stale Xapian docs by treating $over as canonical -sub over_check { - my ($self, $over) = @_; - begin_txn_lazy($self); - my $sth = $over->dbh->prepare(<<''); -SELECT COUNT(*) FROM over WHERE num = ? - - my $xdb = $self->{xdb}; - my $cur = $xdb->postlist_begin(''); - my $end = $xdb->postlist_end(''); - my $xdir = $self->xdir; - for (; $cur != $end; $cur++) { - my $docid = $cur->get_docid; - $sth->execute($docid); - my $x = $sth->fetchrow_array; - next if $x > 0; - warn "I: removing $xdir #$docid, not in `over'\n"; - $xdb->delete_document($docid); - } -} - 1;