+
+ # IMAPTracker is also used for tracking NNTP, UID == article number
+ # LIST.ACTIVE can get the equivalent of UIDVALIDITY, but that's
+ # expensive. So we assume newsgroups don't change:
+ my $itrk = PublicInbox::IMAPTracker->new($url);
+ my (undef, $l_art) = $itrk->get_last;
+ $l_art //= $beg; # initial import
+
+ # allow users to specify articles to refetch
+ # cf. https://tools.ietf.org/id/draft-gilman-news-url-01.txt
+ # nntp://example.com/inbox.foo/$num_a-$num_b
+ $l_art = $num_a if defined($num_a) && $num_a < $l_art;
+ $end = $num_b if defined($num_b) && $num_b < $end;
+
+ return if $l_art >= $end; # nothing to do
+ $beg = $l_art + 1;
+
+ warn "I: $url fetching ARTICLE $beg..$end\n";
+ my $warn_cb = $SIG{__WARN__} || sub { print STDERR @_ };
+ my ($err, $art);
+ local $SIG{__WARN__} = sub {
+ my $pfx = ($_[0] // '') =~ /^([A-Z]: )/g ? $1 : '';
+ $warn_cb->("$pfx$url ", $art ? ("ARTICLE $art") : (), "\n", @_);
+ };
+ my $inboxes = $self->{nntp}->{$url};
+ my $last_art;
+ for ($beg..$end) {
+ last if $self->{quit};
+ $art = $_;
+ my $raw = $nn->article($art);
+ unless (defined($raw)) {
+ my $msg = $nn->message;
+ if ($nn->code == 421) { # pseudo response from Net::Cmd
+ $err = "E: $msg";
+ last;
+ } else { # probably just a deleted message (spam)
+ warn "W: $msg";
+ next;
+ }
+ }
+ s/\r\n/\n/ for @$raw;
+ $raw = join('', @$raw);
+ if (ref($inboxes)) {
+ for my $ibx (@$inboxes) {
+ my $eml = PublicInbox::Eml->new($raw);
+ import_eml($self, $ibx, $eml);
+ }
+ } elsif ($inboxes eq 'watchspam') {
+ my $eml = PublicInbox::Eml->new(\$raw);
+ my $arg = [ $self, $eml, "$url ARTICLE $art" ];
+ $self->{config}->each_inbox(\&remove_eml_i, $arg);
+ } else {
+ die "BUG: destination unknown $inboxes";
+ }
+ $last_art = $art;
+ }
+ _done_for_now($self);
+ $itrk->update_last(0, $last_art);
+ $err;
+}
+
+sub watch_nntp_init ($$) {
+ my ($self, $poll) = @_;
+ eval { require Net::NNTP } or
+ die "Net::NNTP is required for NNTP:\n$@\n";
+ eval { require PublicInbox::IMAPTracker } or
+ die "DBD::SQLite is required for NNTP\n:$@\n";
+
+ my $nn_args = nntp_common_init($self); # read args from config
+
+ # make sure we can connect and cache the credentials in memory
+ $self->{nn_arg} = {}; # schema://authority => Net::NNTP->new args
+ for my $url (sort keys %{$self->{nntp}}) {
+ nn_for($self, $url, $nn_args);
+ }
+ for my $url (keys %{$self->{nntp}}) {
+ my $uri = uri_new($url);
+ my $sec = uri_section($uri);
+ my $intvl = $self->{nntp_opt}->{$sec}->{pollInterval};
+ push @{$poll->{$intvl || 120}}, $url;
+ }
+}
+
+sub watch {
+ my ($self, $sig, $oldset) = @_;
+ $self->{oldset} = $oldset;
+ $self->{sig} = $sig;
+ my $poll = {}; # intvl_seconds => [ url1, url2 ]
+ watch_imap_init($self, $poll) if $self->{imap};
+ watch_nntp_init($self, $poll) if $self->{nntp};
+ while (my ($intvl, $urls) = each %$poll) {
+ # poll all URLs for a given interval sequentially
+ PublicInbox::DS::add_timer(0, \&poll_fetch_fork,
+ [$self, $intvl, $urls]);
+ }
+ watch_fs_init($self) if $self->{mdre};
+ PublicInbox::DS->SetPostLoopCallback(sub { !$self->quit_done });
+ PublicInbox::DS->EventLoop;
+ _done_for_now($self);
+}
+
+sub trigger_scan {
+ my ($self, $op) = @_;
+ push @{$self->{ops}}, $op;
+ PublicInbox::DS::requeue($self);
+}
+
+sub fs_scan_step {
+ my ($self) = @_;