Merge remote-tracking branch 'origin/edit' into next

[public-inbox.git] / lib / PublicInbox / SearchIdx.pm

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 114420e4b82a2b4f71fce48c8538d7ede36e9ea2..7cd67f12f6e5e893d970e82e0e4f1cd03e620127 100644 (file)
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -117,7 +117,11 @@ sub _xdb_acquire {
                }
        }
        return unless defined $flag;
-       $self->{xdb} = Search::Xapian::WritableDatabase->new($dir, $flag);
+       my $xdb = eval { Search::Xapian::WritableDatabase->new($dir, $flag) };
+       if ($@) {
+               die "Failed opening $dir: ", $@;
+       }
+       $self->{xdb} = $xdb;
 }
 
 sub add_val ($$$) {
@@ -542,17 +546,19 @@ sub do_cat_mail {
        $@ ? undef : $mime;
 }
 
+# called by public-inbox-index
 sub index_sync {
        my ($self, $opts) = @_;
+       delete $self->{lock_path} if $opts->{-skip_lock};
        $self->{-inbox}->with_umask(sub { $self->_index_sync($opts) })
 }
 
-sub batch_adjust ($$$$) {
-       my ($max, $bytes, $batch_cb, $latest) = @_;
+sub batch_adjust ($$$$$) {
+       my ($max, $bytes, $batch_cb, $latest, $nr) = @_;
        $$max -= $bytes;
        if ($$max <= 0) {
                $$max = BATCH_BYTES;
-               $batch_cb->($latest);
+               $batch_cb->($nr, $latest);
        }
 }
 
@@ -571,6 +577,7 @@ sub read_log {
        my %D;
        my $line;
        my $newest;
+       my $nr = 0;
        while (defined($line = <$log>)) {
                if ($line =~ /$addmsg/o) {
                        my $blob = $1;
@@ -582,7 +589,7 @@ sub read_log {
                                next;
                        }
                        my $mime = do_cat_mail($git, $blob, \$bytes) or next;
-                       batch_adjust(\$max, $bytes, $batch_cb, $latest);
+                       batch_adjust(\$max, $bytes, $batch_cb, $latest, ++$nr);
                        $add_cb->($self, $mime, $bytes, $blob);
                } elsif ($line =~ /$delmsg/o) {
                        my $blob = $1;
@@ -597,7 +604,7 @@ sub read_log {
                my $mime = do_cat_mail($git, $blob, \$bytes) or next;
                $del_cb->($self, $mime);
        }
-       $batch_cb->($latest, $newest);
+       $batch_cb->($nr, $latest, $newest);
 }
 
 sub _msgmap_init {
@@ -610,7 +617,7 @@ sub _msgmap_init {
 }
 
 sub _git_log {
-       my ($self, $range) = @_;
+       my ($self, $opts, $range) = @_;
        my $git = $self->{git};
 
        if (index($range, '..') < 0) {
@@ -627,12 +634,17 @@ sub _git_log {
        # Count the new files so they can be added newest to oldest
        # and still have numbers increasing from oldest to newest
        my $fcount = 0;
+       my $pr = $opts->{-progress};
+       $pr->("counting changes\n\t$range ... ") if $pr;
        # can't use 'rev-list --count' if we use --diff-filter
        my $fh = $git->popen(qw(log --pretty=tformat:%h
                             --no-notes --no-color --no-renames
                             --diff-filter=AM), $range);
        ++$fcount while <$fh>;
+       close $fh;
        my $high = $self->{mm}->num_highwater;
+       $pr->("$fcount\n") if $pr; # continue previous line
+       $self->{ntodo} = $fcount;
 
        if (index($range, '..') < 0) {
                if ($high && $high == $fcount) {
@@ -686,12 +698,18 @@ sub _last_x_commit {
                $lx = $lm;
        }
        # Use last_commit from msgmap if it is older or unset
-       if (!$lm || ($lx && $lx && is_ancestor($self->{git}, $lm, $lx))) {
+       if (!$lm || ($lx && $lm && is_ancestor($self->{git}, $lm, $lx))) {
                $lx = $lm;
        }
        $lx;
 }
 
+sub reindex_from ($$) {
+       my ($reindex, $last_commit) = @_;
+       return $last_commit unless $reindex;
+       ref($reindex) eq 'HASH' ? $reindex->{from} : '';
+}
+
 # indexes all unindexed messages (v1 only)
 sub _index_sync {
        my ($self, $opts) = @_;
@@ -699,13 +717,14 @@ sub _index_sync {
        my ($last_commit, $lx, $xlog);
        my $git = $self->{git};
        $git->batch_prepare;
+       my $pr = $opts->{-progress};
 
        my $xdb = $self->begin_txn_lazy;
        my $mm = _msgmap_init($self);
        do {
                $xlog = undef;
                $last_commit = _last_x_commit($self, $mm);
-               $lx = $opts->{reindex} ? '' : $last_commit;
+               $lx = reindex_from($opts->{reindex}, $last_commit);
 
                $self->{over}->rollback_lazy;
                $self->{over}->disconnect;
@@ -716,14 +735,14 @@ sub _index_sync {
 
                # ensure we leak no FDs to "git log" with Xapian <= 1.2
                my $range = $lx eq '' ? $tip : "$lx..$tip";
-               $xlog = _git_log($self, $range);
+               $xlog = _git_log($self, $opts, $range);
 
                $xdb = $self->begin_txn_lazy;
        } while (_last_x_commit($self, $mm) ne $last_commit);
 
        my $dbh = $mm->{dbh} if $mm;
        my $cb = sub {
-               my ($commit, $newest) = @_;
+               my ($nr, $commit, $newest) = @_;
                if ($dbh) {
                        if ($newest) {
                                my $cur = $mm->last_commit || '';
@@ -743,6 +762,7 @@ sub _index_sync {
                $git->cleanup;
                $xdb = _xdb_release($self);
                # let another process do some work... <
+               $pr->("indexed $nr/$self->{ntodo}\n") if $pr && $nr;
                if (!$newest) {
                        $xdb = $self->begin_txn_lazy;
                        $dbh->begin_work if $dbh;
@@ -812,6 +832,14 @@ sub commit_txn_lazy {
        delete $self->{txn} or return;
        $self->{-inbox}->with_umask(sub {
                if (my $xdb = $self->{xdb}) {
+
+                       # store 'indexlevel=medium' in v2 part=0 and v1 (only part)
+                       # This metadata is read by Admin::detect_indexlevel:
+                       if (!$self->{partition} # undef or 0, not >0
+                           && $self->{indexlevel} eq 'medium') {
+                               $xdb->set_metadata('indexlevel', 'medium');
+                       }
+
                        $xdb->commit_transaction;
                }
                $self->{over}->commit_lazy if $self->{over};