index: support --fast-noop / -F switch

Note: I'm not sure if it's worth documenting and supporting this
long-term.

We can can avoid taking locks for invocations of "index --all"
and rely on high-resolution ctime (struct timespec st_ctim)
comparisons of msgmap.sqlite3 and the packed-refs + refs/heads
directory of the newest epoch.

This cuts public-inbox-index invocations with
"--all --no-update-extindex -L basic" down from 0.92s to 0.31s.
The change with "-L medium" or "-L full" and (default) non-zero
jobs is even more drastic, reducing a 12-13s no-op invocation
down to the same 0.31s

diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index 531a72b28abd5d8bb101839662c352d27fa3d548..2b849ddffb0d1221a1d20a2829823174f7cb0d4a 100644 (file)
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -1351,11 +1351,19 @@ sub index_sync {
        $opt //= {};
        return xapian_only($self, $opt) if $opt->{xapian_only};
 
-       my $pr = $opt->{-progress};
        my $epoch_max;
-       my $latest = $self->{ibx}->git_dir_latest(\$epoch_max);
-       return unless defined $latest;
+       my $latest = $self->{ibx}->git_dir_latest(\$epoch_max) // return;
+       if ($opt->{'fast-noop'}) { # nanosecond (st_ctim) comparison
+               use Time::HiRes qw(stat);
+               if (my @mm = stat("$self->{ibx}->{inboxdir}/msgmap.sqlite3")) {
+                       my $c = $mm[10]; # 10 = ctime (nsec NV)
+                       my @hd = stat("$latest/refs/heads");
+                       my @pr = stat("$latest/packed-refs");
+                       return if $c > ($hd[10] // 0) && $c > ($pr[10] // 0);
+               }
+       }
 
+       my $pr = $opt->{-progress};
        my $seq = $opt->{sequential_shard};
        my $art_beg; # the NNTP article number we start xapian_only at
        my $idxlevel = $self->{ibx}->{indexlevel};

diff --git a/script/public-inbox-index b/script/public-inbox-index
index f10bb5adb4110ad69552cfa7a221b09a26d3ec48..91afac88a23f7af2cd63ec12a5815605ec15a495 100755 (executable)
--- a/script/public-inbox-index
+++ b/script/public-inbox-index
@@ -42,7 +42,7 @@ GetOptions($opt, qw(verbose|v+ reindex rethread compact|c+ jobs|j=i prune
                batch_size|batch-size=s
                sequential_shard|seq-shard|sequential-shard
                no-update-extindex update-extindex|E=s@
-               skip-docdata all help|h))
+               fast-noop|F skip-docdata all help|h))
        or die $help;
 if ($opt->{help}) { print $help; exit 0 };
 die "--jobs must be >= 0\n" if defined $opt->{jobs} && $opt->{jobs} < 0;