Upon startup, it scans the mailbox for new messages to be
imported while it was not running.
-Currently, only Maildirs are supported and the
-L<Filesys::Notify::Simple> Perl module is required.
+Currently, only Maildirs are supported.
For now, IMAP users should use tools such as L<mbsync(1)>
or L<offlineimap(1)> to bidirectionally sync their IMAP
(optional for stale FD cleanup in daemons,
typically installed alongside Perl5)
-- Filesys::Notify::Simple deb: libfilesys-notify-simple-perl
- pkg: p5-Filesys-Notify-Simple
- rpm: perl-Filesys-Notify-Simple
- (for public-inbox-watch, pulled in by Plack)
-
- Linux::Inotify2 deb: liblinux-inotify2-perl
rpm: perl-Linux-Inotify2
(for public-inbox-watch on Linux)
-- Filesys::Notify::KQueue pkg: p5-Filesys-Notify-KQueue
- (for public-inbox-watch on FreeBSD)
-
- IO::Compress (::Gzip) deb: perl-modules (or libio-compress-perl)
pkg: perl5
rpm: perl-IO-Compress
lib/PublicInbox/DSKQXS.pm
lib/PublicInbox/DSPoll.pm
lib/PublicInbox/Daemon.pm
+lib/PublicInbox/DirIdle.pm
lib/PublicInbox/DummyInbox.pm
lib/PublicInbox/Emergency.pm
lib/PublicInbox/Eml.pm
t/convert-compact.t
t/data/0001.patch
t/data/message_embed.eml
+t/dir_idle.t
t/ds-kqxs.t
t/ds-leak.t
t/ds-poll.t
# Plack is needed for public-inbox-httpd and PublicInbox::WWW
# 'Plack' => 0,
- # Filesys::Notify::Simple is pulled in by Plack, but also
- # needed by public-inbox-watch (for now)
- # 'Filesys::Notify::Simple' => 0,
-
# TODO: this should really be made optional...
'URI::Escape' => 0,
BSD::Resource
DBD::SQLite
DBI
- Filesys::Notify::Simple
Inline::C
Net::Server
Plack
--- /dev/null
+# Copyright (C) 2020 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# Used by public-inbox-watch for Maildir (and possibly MH in the future)
+package PublicInbox::DirIdle;
+use strict;
+use base 'PublicInbox::DS';
+use fields qw(inot);
+use PublicInbox::Syscall qw(EPOLLIN EPOLLET);
+use PublicInbox::In2Tie;
+
+my ($MAIL_IN, $ino_cls);
+if ($^O eq 'linux' && eval { require Linux::Inotify2; 1 }) {
+ $MAIL_IN = Linux::Inotify2::IN_MOVED_TO() |
+ Linux::Inotify2::IN_CREATE();
+ $ino_cls = 'Linux::Inotify2';
+} elsif (eval { require PublicInbox::KQNotify }) {
+ $MAIL_IN = PublicInbox::KQNotify::MOVED_TO_OR_CREATE();
+ $ino_cls = 'PublicInbox::KQNotify';
+} else {
+ require PublicInbox::FakeInotify;
+ $MAIL_IN = PublicInbox::FakeInotify::MOVED_TO_OR_CREATE();
+}
+
+sub new {
+ my ($class, $dirs, $cb) = @_;
+ my $self = fields::new($class);
+ my $inot;
+ if ($ino_cls) {
+ $inot = $ino_cls->new or die "E: $ino_cls->new: $!";
+ my $io = PublicInbox::In2Tie::io($inot);
+ $self->SUPER::new($io, EPOLLIN | EPOLLET);
+ } else {
+ require PublicInbox::FakeInotify;
+ $inot = PublicInbox::FakeInotify->new; # starts timer
+ }
+
+ # Linux::Inotify2->watch or similar
+ $inot->watch($_, $MAIL_IN, $cb) for @$dirs;
+ $self->{inot} = $inot;
+ $self;
+}
+
+sub event_step {
+ my ($self) = @_;
+ eval { $self->{inot}->poll }; # Linux::Inotify2::poll
+ warn "$self->{inot}->poll err: $@\n" if $@;
+}
+
+1;
# on Linux::Inotify2 objects
package PublicInbox::In2Tie;
use strict;
+use Symbol qw(gensym);
+
+sub io {
+ my $in2 = $_[0];
+ $in2->blocking(0);
+ if ($in2->can('on_overflow')) {
+ # broadcasts everything on overflow
+ $in2->on_overflow(undef);
+ }
+ my $io = gensym;
+ tie *$io, __PACKAGE__, $in2;
+ $io;
+}
sub TIEHANDLE {
my ($class, $in2) = @_;
use base qw(PublicInbox::DS);
use fields qw(pi_config inot pathmap);
use Cwd qw(abs_path);
-use Symbol qw(gensym);
use PublicInbox::Syscall qw(EPOLLIN EPOLLET);
my $IN_MODIFY = 0x02; # match Linux inotify
my $ino_cls;
my $inot;
if ($ino_cls) {
$inot = $ino_cls->new or die "E: $ino_cls->new: $!";
- my $sock = gensym;
- tie *$sock, 'PublicInbox::In2Tie', $inot;
- $inot->blocking(0);
- if ($inot->can('on_overflow')) {
- # broadcasts everything on overflow
- $inot->on_overflow(undef);
- }
- $self->SUPER::new($sock, EPOLLIN | EPOLLET);
+ my $io = PublicInbox::In2Tie::io($inot);
+ $self->SUPER::new($io, EPOLLIN | EPOLLET);
} else {
require PublicInbox::FakeInotify;
$inot = PublicInbox::FakeInotify->new;
use POSIX qw(dup2);
use IO::Socket::INET;
our @EXPORT = qw(tmpdir tcp_server tcp_connect require_git require_mods
- run_script start_script key2sub xsys xqx eml_load);
+ run_script start_script key2sub xsys xqx eml_load tick);
sub eml_load ($) {
my ($path, $cb) = @_;
$self->join('TERM');
}
+package PublicInbox::TestCommon::InboxWakeup;
+use strict;
+sub on_inbox_unlock { ${$_[0]}->($_[1]) }
+
1;
}
}
-sub _try_fsn_paths {
- my ($self, $scan_re, $paths) = @_;
- foreach (@$paths) {
- my $path = $_->{path};
- if ($path =~ $scan_re) {
- scan($self, $path);
- } else {
- _try_path($self, $path);
- }
- }
- _done_for_now($self);
-}
-
sub remove_eml_i { # each_inbox callback
my ($ibx, $arg) = @_;
my ($self, $eml, $loc) = @$arg;
sub watch_fs {
my ($self) = @_;
+ require PublicInbox::DirIdle;
my $scan = File::Temp->newdir("public-inbox-watch.$$.scan.XXXXXX",
TMPDIR => 1);
my $scandir = $self->{scandir} = $scan->dirname;
- my $re = qr!\A$scandir/!;
- my $cb = sub { _try_fsn_paths($self, $re, \@_) };
-
- eval { require Filesys::Notify::Simple } or
- die "Filesys::Notify::Simple is currently required for $0\n";
- my $fsn = Filesys::Notify::Simple->new([@{$self->{mdir}}, $scandir]);
- $fsn->wait($cb) until $self->{quit};
+ my $scan_re = qr!\A$scandir/!;
+ my $done = sub {
+ delete $self->{done_timer};
+ _done_for_now($self);
+ };
+ my $cb = sub {
+ my $path = $_[0]->fullname;
+ if ($path =~ $scan_re) {
+ scan($self, $path);
+ } else {
+ _try_path($self, $path);
+ }
+ $self->{done_timer} //= PublicInbox::DS::requeue($done);
+ };
+ my $di = PublicInbox::DirIdle->new([@{$self->{mdir}}, $scandir], $cb);
+ PublicInbox::DS->SetPostLoopCallback(sub { !$self->{quit} });
+ PublicInbox::DS->EventLoop;
+ _done_for_now($self);
}
# returns the git config section name, e.g [imap "imaps://user@example.com"]
$watch_md->quit if $watch_md;
$watch_md = undef;
};
- alarm(1);
+ # --no-scan is only intended for testing atm, undocumented.
+ alarm(1) unless (grep(/\A--no-scan\z/, @ARGV));
$watch_md->watch while ($watch_md);
}
--- /dev/null
+#!perl -w
+# Copyright (C) 2020 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use Test::More;
+use_ok 'PublicInbox::DirIdle';
+done_testing;
PublicInbox::DS->Reset;
my $ii = PublicInbox::InboxIdle->new($cfg);
my $cb = sub { PublicInbox::DS->SetPostLoopCallback(sub {}) };
- my $obj = bless \$cb, 'InboxWakeup';
+ my $obj = bless \$cb, 'PublicInbox::TestCommon::InboxWakeup';
$cfg->each_inbox(sub { $_[0]->subscribe_unlock('ident', $obj) });
open my $err, '+>', undef or BAIL_OUT $!;
my $w = start_script(['-watch'], undef, { 2 => $err });
unlike($eout, qr/uninitialized/i, 'no uninitialized warnings');
done_testing;
-
-package InboxWakeup;
-use strict;
-sub on_inbox_unlock { ${$_[0]}->() }
use Test::More;
use PublicInbox::Eml;
use PublicInbox::Config;
-require_mods(qw(Filesys::Notify::Simple DBD::SQLite Search::Xapian));
+require_mods(qw(DBD::SQLite Search::Xapian));
use_ok 'PublicInbox::WatchMaildir';
use_ok 'PublicInbox::Emergency';
my ($tmpdir, $for_destroy) = tmpdir();
use PublicInbox::Config;
use PublicInbox::TestCommon;
use PublicInbox::Import;
-require_mods(qw(Filesys::Notify::Simple));
my ($tmpdir, $for_destroy) = tmpdir();
my $git_dir = "$tmpdir/test.git";
my $maildir = "$tmpdir/md";
'only got the spam folder to watch');
}
-my $config = PublicInbox::Config->new(\<<EOF);
-$cfgpfx.address=$addr
-$cfgpfx.inboxdir=$git_dir
-$cfgpfx.watch=maildir:$maildir
-$cfgpfx.filter=PublicInbox::Filter::Vger
-publicinboxlearn.watchspam=maildir:$spamdir
+my $cfg_path = "$tmpdir/config";
+{
+ open my $fh, '>', $cfg_path or BAIL_OUT $!;
+ print $fh <<EOF or BAIL_OUT $!;
+[publicinbox "test"]
+ address = $addr
+ inboxdir = $git_dir
+ watch = maildir:$maildir
+ filter = PublicInbox::Filter::Vger
+[publicinboxlearn]
+ watchspam = maildir:$spamdir
EOF
+ close $fh or BAIL_OUT $!;
+}
+my $config = PublicInbox::Config->new($cfg_path);
PublicInbox::WatchMaildir->new($config)->scan('full');
my $git = PublicInbox::Git->new($git_dir);
my @list = $git->qx(qw(rev-list refs/heads/master));
like($$mref, qr/something\n\z/s, 'message scrubbed on import');
}
+# end-to-end test which actually uses inotify/kevent
+{
+ my $env = { PI_CONFIG => $cfg_path };
+ $git->cleanup;
+
+ # n.b. --no-scan is only intended for testing atm
+ my $wm = start_script([qw(-watch --no-scan)], $env);
+ my $eml = eml_load('t/data/0001.patch');
+ $eml->header_set('Cc', $addr);
+ my $em = PublicInbox::Emergency->new($maildir);
+ $em->prepare(\($eml->as_string));
+
+ use_ok 'PublicInbox::InboxIdle';
+ use_ok 'PublicInbox::DS';
+ my $delivered = 0;
+ my $cb = sub {
+ my ($ibx) = @_;
+ diag "message delivered to `$ibx->{name}'";
+ $delivered++;
+ };
+ PublicInbox::DS->Reset;
+ my $ii = PublicInbox::InboxIdle->new($config);
+ my $obj = bless \$cb, 'PublicInbox::TestCommon::InboxWakeup';
+ $config->each_inbox(sub { $_[0]->subscribe_unlock('ident', $obj) });
+ PublicInbox::DS->SetPostLoopCallback(sub { $delivered == 0 });
+
+ # wait for -watch to setup inotify watches
+ my $sleep = 1;
+ if (eval { require Linux::Inotify2 } && -d "/proc/$wm->{pid}/fd") {
+ my $end = time + 2;
+ my (@ino, @ino_info);
+ do {
+ @ino = grep {
+ (readlink($_)//'') =~ /\binotify\b/
+ } glob("/proc/$wm->{pid}/fd/*");
+ } until (@ino || time > $end || !tick);
+ if (scalar(@ino) == 1) {
+ my $ino_fd = (split('/', $ino[0]))[-1];
+ my $ino_fdinfo = "/proc/$wm->{pid}/fdinfo/$ino_fd";
+ while (time < $end && open(my $fh, '<', $ino_fdinfo)) {
+ @ino_info = grep(/^inotify wd:/, <$fh>);
+ last if @ino_info >= 4;
+ tick;
+ }
+ $sleep = undef if @ino_info >= 4;
+ }
+ }
+ if ($sleep) {
+ diag "waiting ${sleep}s for -watch to start up";
+ sleep $sleep;
+ }
+
+ $em->commit; # wake -watch up
+ diag 'waiting for -watch to import new message';
+ PublicInbox::DS->EventLoop;
+ $wm->kill;
+ $wm->join;
+ $ii->close;
+ PublicInbox::DS->Reset;
+ my $head = $git->qx(qw(cat-file commit HEAD));
+ my $subj = $eml->header('Subject');
+ like($head, qr/^\Q$subj\E/sm, 'new commit made');
+}
+
sub is_maildir {
my ($dir) = @_;
PublicInbox::WatchMaildir::is_maildir($dir);
use PublicInbox::TestCommon;
use PublicInbox::Import;
require_git(2.6);
-require_mods(qw(Search::Xapian DBD::SQLite Filesys::Notify::Simple));
+require_mods(qw(Search::Xapian DBD::SQLite));
require PublicInbox::V2Writable;
my ($tmpdir, $for_destroy) = tmpdir();
my $inboxdir = "$tmpdir/v2";
use PublicInbox::Config;
use PublicInbox::TestCommon;
require_git(2.6);
-require_mods(qw(Search::Xapian DBD::SQLite Filesys::Notify::Simple));
+require_mods(qw(Search::Xapian DBD::SQLite));
my ($tmpdir, $for_destroy) = tmpdir();
my $inboxdir = "$tmpdir/v2";
my $maildir = "$tmpdir/md";