lock: reduce inotify wakeups

We can reduce the amount of platform-specific code by always
relying on IN_MODIFY/NOTE_WRITE notifications from lock release.
This reduces the number of times our read-only daemons will
need to wake up when -watch sees no-op message changes
(e.g. replied, seen, recent flag changes).

diff --git a/lib/PublicInbox/Import.pm b/lib/PublicInbox/Import.pm
index af35905be4964b87ad9ea5dcb7eb78b7fe3e9d9a..ae508cd80132ed496fe2ed1c627df7e3246e7d66 100644 (file)
--- a/lib/PublicInbox/Import.pm
+++ b/lib/PublicInbox/Import.pm
@@ -468,9 +468,9 @@ sub done {
        waitpid($pid, 0) == $pid or die 'fast-import did not finish';
        $? == 0 or die "fast-import failed: $?";
 
-       _update_git_info($self, 1) if delete $self->{nchg};
-
-       $self->lock_release;
+       my $nchg = delete $self->{nchg};
+       _update_git_info($self, 1) if $nchg;
+       $self->lock_release(!!$nchg);
 
        $self->{git}->cleanup;
 }

diff --git a/lib/PublicInbox/InboxIdle.pm b/lib/PublicInbox/InboxIdle.pm
index d0bb43c58486a5900348a74031cb44f95820bf74..97e9d53250eb43285ae7eba56c3f4ae79d898d8e 100644 (file)
--- a/lib/PublicInbox/InboxIdle.pm
+++ b/lib/PublicInbox/InboxIdle.pm
@@ -8,13 +8,13 @@ use fields qw(pi_config inot pathmap);
 use Cwd qw(abs_path);
 use Symbol qw(gensym);
 use PublicInbox::Syscall qw(EPOLLIN EPOLLET);
-my $IN_CLOSE = 0x08 | 0x10; # match Linux inotify
+my $IN_MODIFY = 0x02; # match Linux inotify
 my $ino_cls;
 if ($^O eq 'linux' && eval { require Linux::Inotify2; 1 }) {
-       $IN_CLOSE = Linux::Inotify2::IN_CLOSE();
+       $IN_MODIFY = Linux::Inotify2::IN_MODIFY();
        $ino_cls = 'Linux::Inotify2';
 } elsif (eval { require PublicInbox::KQNotify }) {
-       $IN_CLOSE = PublicInbox::KQNotify::IN_CLOSE();
+       $IN_MODIFY = PublicInbox::KQNotify::NOTE_WRITE();
        $ino_cls = 'PublicInbox::KQNotify';
 }
 require PublicInbox::In2Tie if $ino_cls;
@@ -39,7 +39,7 @@ sub in2_arm ($$) { # PublicInbox::Config::each_inbox callback
        $cur->[0] = $ibx;
 
        my $lock = "$dir/".($ibx->version >= 2 ? 'inbox.lock' : 'ssoma.lock');
-       $cur->[1] = $inot->watch($lock, $IN_CLOSE, sub { $ibx->on_unlock });
+       $cur->[1] = $inot->watch($lock, $IN_MODIFY, sub { $ibx->on_unlock });
 
        # TODO: detect deleted packs (and possibly other files)
 }

diff --git a/lib/PublicInbox/KQNotify.pm b/lib/PublicInbox/KQNotify.pm
index 1b5c578ef3df8aef15a9adefab55c00c11ded432..110594cc02c0332695831ebe2e3a273ee2da10bd 100644 (file)
--- a/lib/PublicInbox/KQNotify.pm
+++ b/lib/PublicInbox/KQNotify.pm
@@ -8,10 +8,6 @@ use strict;
 use IO::KQueue;
 use PublicInbox::DSKQXS; # wraps IO::KQueue for fork-safe DESTROY
 
-# only true as far as public-inbox is concerned with .lock files:
-sub IN_CLOSE () { NOTE_WRITE }
-#sub IN_CLOSE () { 0x200 } # NOTE_CLOSE_WRITE (FreeBSD 11+ only)
-
 sub new {
        my ($class) = @_;
        bless { dskq => PublicInbox::DSKQXS->new, watch => {} }, $class;
@@ -26,7 +22,7 @@ sub watch {
                EV_ADD | EV_CLEAR, # flags
                $mask, # fflags
                0, 0); # data, udata
-       if ($mask == IN_CLOSE) {
+       if ($mask == NOTE_WRITE) {
                $self->{watch}->{$ident} = [ $fh, $cb ];
        } else {
                die "TODO Not implemented: $mask";
@@ -52,7 +48,7 @@ sub poll {
        for my $kev (@kevents) {
                my $ident = $kev->[KQ_IDENT];
                my $mask = $kev->[KQ_FFLAGS];
-               if (($mask & IN_CLOSE) == IN_CLOSE) {
+               if (($mask & NOTE_WRITE) == NOTE_WRITE) {
                        eval { $self->{watch}->{$ident}->[1]->() };
                }
        }

diff --git a/lib/PublicInbox/Lock.pm b/lib/PublicInbox/Lock.pm
index 693a37949a651f7935df09d6c58a785dc2322140..c0d4d3b35c4a4bbaa5d90c4273eb6cfcc383b6be 100644 (file)
--- a/lib/PublicInbox/Lock.pm
+++ b/lib/PublicInbox/Lock.pm
@@ -21,16 +21,12 @@ sub lock_acquire {
 }
 
 sub lock_release {
-       my ($self) = @_;
+       my ($self, $wake) = @_;
        return unless $self->{lock_path};
        my $lockfh = delete $self->{lockfh} or croak 'not locked';
 
-       # NetBSD 8.1 and OpenBSD 6.5 (and maybe other versions/*BSDs) lack
-       # NOTE_CLOSE_WRITE from FreeBSD 11+, so trigger NOTE_WRITE, instead.
-       # We also need to change the ctime on Linux systems w/o inotify
-       if ($^O ne 'linux' || !eval { require Linux::Inotify2; 1 }) {
-               syswrite($lockfh, '.');
-       }
+       syswrite($lockfh, '.') if $wake;
+
        flock($lockfh, LOCK_UN) or die "unlock failed: $!\n";
        close $lockfh or die "close failed: $!\n";
 }

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 00e63938fc068f6eb6c8e0199fc16130f96426d4..4caa66d3751d175f1a22e6685b173b66e800dec6 100644 (file)
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -78,12 +78,12 @@ sub new {
 sub need_xapian ($) { $_[0]->{indexlevel} =~ $xapianlevels }
 
 sub _xdb_release {
-       my ($self) = @_;
+       my ($self, $wake) = @_;
        if (need_xapian($self)) {
                my $xdb = delete $self->{xdb} or croak 'not acquired';
                $xdb->close;
        }
-       $self->lock_release if $self->{creat};
+       $self->lock_release($wake) if $self->{creat};
        undef;
 }
 
@@ -800,7 +800,7 @@ sub _index_sync {
                }
                $self->commit_txn_lazy;
                $git->cleanup;
-               $xdb = _xdb_release($self);
+               $xdb = _xdb_release($self, $nr);
                # let another process do some work... <
                $pr->("indexed $nr/$self->{ntodo}\n") if $pr && $nr;
                if (!$newest) {

diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index a0f041dd3dc0d287918142a39ceecff2b0abda3d..8b31b69a62f7fb01f8525ae997c9bc05c74330bf 100644 (file)
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -113,6 +113,7 @@ sub new {
                im => undef, #  PublicInbox::Import
                parallel => 1,
                transact_bytes => 0,
+               total_bytes => 0,
                current_info => '',
                xpfx => $xpfx,
                over => PublicInbox::OverIdx->new("$xpfx/over.sqlite3", 1),
@@ -659,6 +660,7 @@ sub checkpoint ($;$) {
 
                $dbh->begin_work;
        }
+       $self->{total_bytes} += $self->{transact_bytes};
        $self->{transact_bytes} = 0;
 }
 
@@ -681,8 +683,9 @@ sub done {
        }
        $self->{over}->disconnect;
        delete $self->{bnote};
-       $self->{transact_bytes} = 0;
-       $self->lock_release if $shards;
+       my $nbytes = $self->{total_bytes};
+       $self->{total_bytes} = 0;
+       $self->lock_release(!!$nbytes) if $shards;
        $self->{-inbox}->git->cleanup;
 }
 

diff --git a/lib/PublicInbox/Xapcmd.pm b/lib/PublicInbox/Xapcmd.pm
index 337978bd1a8799b79d10a315341333b06cf6ae3c..784140416e7fe5bb0ff3d0dfd86b24c28ba601da 100644 (file)
--- a/lib/PublicInbox/Xapcmd.pm
+++ b/lib/PublicInbox/Xapcmd.pm
@@ -39,6 +39,7 @@ sub commit_changes ($$$$) {
                        my $tmp_over = "$new/over.sqlite3";
                        $over->connect->sqlite_backup_to_file($tmp_over);
                        $over = undef;
+                       syswrite($im->{lockfh}, '.'); # trigger ->check_inodes
                }
 
                if (!defined($new)) { # culled shard