Merge remote-tracking branch 'origin/edit' into next

[public-inbox.git] / lib / PublicInbox / SearchIdx.pm

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 0aeeb6bc544acce630adacd28e6b063e8295b6be..7cd67f12f6e5e893d970e82e0e4f1cd03e620127 100644 (file)
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -117,7 +117,11 @@ sub _xdb_acquire {
                }
        }
        return unless defined $flag;
-       $self->{xdb} = Search::Xapian::WritableDatabase->new($dir, $flag);
+       my $xdb = eval { Search::Xapian::WritableDatabase->new($dir, $flag) };
+       if ($@) {
+               die "Failed opening $dir: ", $@;
+       }
+       $self->{xdb} = $xdb;
 }
 
 sub add_val ($$$) {
@@ -549,12 +553,12 @@ sub index_sync {
        $self->{-inbox}->with_umask(sub { $self->_index_sync($opts) })
 }
 
-sub batch_adjust ($$$$) {
-       my ($max, $bytes, $batch_cb, $latest) = @_;
+sub batch_adjust ($$$$$) {
+       my ($max, $bytes, $batch_cb, $latest, $nr) = @_;
        $$max -= $bytes;
        if ($$max <= 0) {
                $$max = BATCH_BYTES;
-               $batch_cb->($latest);
+               $batch_cb->($nr, $latest);
        }
 }
 
@@ -573,6 +577,7 @@ sub read_log {
        my %D;
        my $line;
        my $newest;
+       my $nr = 0;
        while (defined($line = <$log>)) {
                if ($line =~ /$addmsg/o) {
                        my $blob = $1;
@@ -584,7 +589,7 @@ sub read_log {
                                next;
                        }
                        my $mime = do_cat_mail($git, $blob, \$bytes) or next;
-                       batch_adjust(\$max, $bytes, $batch_cb, $latest);
+                       batch_adjust(\$max, $bytes, $batch_cb, $latest, ++$nr);
                        $add_cb->($self, $mime, $bytes, $blob);
                } elsif ($line =~ /$delmsg/o) {
                        my $blob = $1;
@@ -599,7 +604,7 @@ sub read_log {
                my $mime = do_cat_mail($git, $blob, \$bytes) or next;
                $del_cb->($self, $mime);
        }
-       $batch_cb->($latest, $newest);
+       $batch_cb->($nr, $latest, $newest);
 }
 
 sub _msgmap_init {
@@ -612,7 +617,7 @@ sub _msgmap_init {
 }
 
 sub _git_log {
-       my ($self, $range) = @_;
+       my ($self, $opts, $range) = @_;
        my $git = $self->{git};
 
        if (index($range, '..') < 0) {
@@ -629,12 +634,17 @@ sub _git_log {
        # Count the new files so they can be added newest to oldest
        # and still have numbers increasing from oldest to newest
        my $fcount = 0;
+       my $pr = $opts->{-progress};
+       $pr->("counting changes\n\t$range ... ") if $pr;
        # can't use 'rev-list --count' if we use --diff-filter
        my $fh = $git->popen(qw(log --pretty=tformat:%h
                             --no-notes --no-color --no-renames
                             --diff-filter=AM), $range);
        ++$fcount while <$fh>;
+       close $fh;
        my $high = $self->{mm}->num_highwater;
+       $pr->("$fcount\n") if $pr; # continue previous line
+       $self->{ntodo} = $fcount;
 
        if (index($range, '..') < 0) {
                if ($high && $high == $fcount) {
@@ -688,7 +698,7 @@ sub _last_x_commit {
                $lx = $lm;
        }
        # Use last_commit from msgmap if it is older or unset
-       if (!$lm || ($lx && $lx && is_ancestor($self->{git}, $lm, $lx))) {
+       if (!$lm || ($lx && $lm && is_ancestor($self->{git}, $lm, $lx))) {
                $lx = $lm;
        }
        $lx;
@@ -707,6 +717,7 @@ sub _index_sync {
        my ($last_commit, $lx, $xlog);
        my $git = $self->{git};
        $git->batch_prepare;
+       my $pr = $opts->{-progress};
 
        my $xdb = $self->begin_txn_lazy;
        my $mm = _msgmap_init($self);
@@ -724,14 +735,14 @@ sub _index_sync {
 
                # ensure we leak no FDs to "git log" with Xapian <= 1.2
                my $range = $lx eq '' ? $tip : "$lx..$tip";
-               $xlog = _git_log($self, $range);
+               $xlog = _git_log($self, $opts, $range);
 
                $xdb = $self->begin_txn_lazy;
        } while (_last_x_commit($self, $mm) ne $last_commit);
 
        my $dbh = $mm->{dbh} if $mm;
        my $cb = sub {
-               my ($commit, $newest) = @_;
+               my ($nr, $commit, $newest) = @_;
                if ($dbh) {
                        if ($newest) {
                                my $cur = $mm->last_commit || '';
@@ -751,6 +762,7 @@ sub _index_sync {
                $git->cleanup;
                $xdb = _xdb_release($self);
                # let another process do some work... <
+               $pr->("indexed $nr/$self->{ntodo}\n") if $pr && $nr;
                if (!$newest) {
                        $xdb = $self->begin_txn_lazy;
                        $dbh->begin_work if $dbh;
@@ -820,6 +832,14 @@ sub commit_txn_lazy {
        delete $self->{txn} or return;
        $self->{-inbox}->with_umask(sub {
                if (my $xdb = $self->{xdb}) {
+
+                       # store 'indexlevel=medium' in v2 part=0 and v1 (only part)
+                       # This metadata is read by Admin::detect_indexlevel:
+                       if (!$self->{partition} # undef or 0, not >0
+                           && $self->{indexlevel} eq 'medium') {
+                               $xdb->set_metadata('indexlevel', 'medium');
+                       }
+
                        $xdb->commit_transaction;
                }
                $self->{over}->commit_lazy if $self->{over};