use v5.10.1;
use parent qw(Exporter PublicInbox::IPC);
use PublicInbox::Eml;
-
our %IMAPflags2kw = map {; "\\\u$_" => $_ } qw(seen answered flagged draft);
+$IMAPflags2kw{'$Forwarded'} = 'forwarded'; # RFC 5550
+
+our @EXPORT = qw(uri_section imap_uri nntp_uri);
-# TODO: trim this down, this is huge
-our @EXPORT = qw(uri_new uri_section
- nn_new imap_uri nntp_uri
- cfg_bool cfg_intvl imap_common_init nntp_common_init
- );
+sub ndump {
+ require Data::Dumper;
+ Data::Dumper->new(\@_)->Useqq(1)->Terse(1)->Dump;
+}
# returns the git config section name, e.g [imap "imaps://user@example.com"]
# without the mailbox, so we can share connections between different inboxes
$mic;
}
-sub uri_new {
- my ($url) = @_;
- require URI;
-
- # URI::snews exists, URI::nntps does not, so use URI::snews
- $url =~ s!\Anntps://!snews://!i;
- URI->new($url);
-}
-
# Net::NNTP doesn't support CAPABILITIES, yet
sub try_starttls ($) {
my ($host) = @_;
$self->{quiet} = 1 if $lei && $lei->{opt}->{quiet};
eval { require PublicInbox::IMAPClient } or
die "Mail::IMAPClient is required for IMAP:\n$@\n";
- eval { require PublicInbox::IMAPTracker } or
+ ($lei || eval { require PublicInbox::IMAPTracker }) or
die "DBD::SQLite is required for IMAP\n:$@\n";
require PublicInbox::URIimap;
my $cfg = $self->{pi_cfg} // $lei->_lei_cfg;
my $mics = {}; # schema://authority => IMAPClient obj
for my $uri (@{$self->{imap_order}}) {
my $sec = uri_section($uri);
- $mics->{$sec} //= mic_for($self, "$sec/", $mic_args, $lei);
+ my $mic = $mics->{$sec} //=
+ mic_for($self, "$sec/", $mic_args, $lei) //
+ die "Unable to continue\n";
next unless $self->isa('PublicInbox::NetWriter');
my $dst = $uri->mailbox // next;
- my $mic = $mics->{$sec};
next if $mic->exists($dst); # already exists
$mic->create($dst) or die "CREATE $dst failed <$uri>: $@";
}
$self->{quiet} = 1 if $lei && $lei->{opt}->{quiet};
eval { require Net::NNTP } or
die "Net::NNTP is required for NNTP:\n$@\n";
- eval { require PublicInbox::IMAPTracker } or
+ ($lei || eval { require PublicInbox::IMAPTracker }) or
die "DBD::SQLite is required for NNTP\n:$@\n";
my $cfg = $self->{pi_cfg} // $lei->_lei_cfg;
my $nn_args = {}; # scheme://authority => Net::NNTP->new arg
}
sub _imap_do_msg ($$$$$) {
- my ($self, $uri, $uid, $raw, $flags) = @_;
+ my ($self, $url, $uid, $raw, $flags) = @_;
# our target audience expects LF-only, save storage
$$raw =~ s/\r\n/\n/sg;
my $kw = [];
for my $f (split(/ /, $flags)) {
- my $k = $IMAPflags2kw{$f} // next; # TODO: X-Label?
- push @$kw, $k;
+ if (my $k = $IMAPflags2kw{$f}) {
+ push @$kw, $k;
+ } elsif ($f eq "\\Recent") { # not in JMAP
+ } elsif ($f eq "\\Deleted") { # not in JMAP
+ return;
+ } elsif ($self->{verbose}) {
+ warn "# unknown IMAP flag $f <$url/;UID=$uid>\n";
+ }
}
+ @$kw = sort @$kw; # for all UI/UX purposes
my ($eml_cb, @args) = @{$self->{eml_each}};
- $eml_cb->($uri, $uid, $kw, PublicInbox::Eml->new($raw), @args);
+ $eml_cb->($url, $uid, $kw, PublicInbox::Eml->new($raw), @args);
+}
+
+sub run_commit_cb ($) {
+ my ($self) = @_;
+ my $cmt_cb_args = $self->{on_commit} or return;
+ my ($cb, @args) = @$cmt_cb_args;
+ $cb->(@args);
+}
+
+sub _itrk_last ($$;$) {
+ my ($self, $uri, $r_uidval) = @_;
+ return (undef, undef, $r_uidval) unless $self->{incremental};
+ my ($itrk, $l_uid, $l_uidval);
+ if (defined(my $lms = $self->{-lms_ro})) { # LeiMailSync or 0
+ $uri->uidvalidity($r_uidval) if defined $r_uidval;
+ my $x;
+ $l_uid = ($lms && ($x = $lms->location_stats($$uri))) ?
+ $x->{'uid.max'} : undef;
+ # itrk remains undef, lei/store worker writes to
+ # mail_sync.sqlite3
+ } else {
+ $itrk = PublicInbox::IMAPTracker->new($$uri);
+ ($l_uidval, $l_uid) = $itrk->get_last($$uri);
+ }
+ ($itrk, $l_uid, $l_uidval //= $r_uidval);
}
sub _imap_fetch_all ($$$) {
- my ($self, $mic, $uri) = @_;
- my $sec = uri_section($uri);
- my $mbx = $uri->mailbox;
+ my ($self, $mic, $orig_uri) = @_;
+ my $sec = uri_section($orig_uri);
+ my $mbx = $orig_uri->mailbox;
$mic->Clear(1); # trim results history
$mic->examine($mbx) or return "E: EXAMINE $mbx ($sec) failed: $!";
my ($r_uidval, $r_uidnext);
last if $r_uidval && $r_uidnext;
}
$r_uidval //= $mic->uidvalidity($mbx) //
- return "E: $uri cannot get UIDVALIDITY";
+ return "E: $orig_uri cannot get UIDVALIDITY";
$r_uidnext //= $mic->uidnext($mbx) //
- return "E: $uri cannot get UIDNEXT";
- my $itrk = $self->{incremental} ?
- PublicInbox::IMAPTracker->new($$uri) : 0;
- my ($l_uidval, $l_uid) = $itrk ? $itrk->get_last : ();
- $l_uidval //= $r_uidval; # first time
+ return "E: $orig_uri cannot get UIDNEXT";
+ my $uri = $orig_uri->clone;
+ my ($itrk, $l_uid, $l_uidval) = _itrk_last($self, $uri, $r_uidval);
+ return <<EOF if $l_uidval != $r_uidval;
+E: $uri UIDVALIDITY mismatch
+E: local=$l_uidval != remote=$r_uidval
+EOF
+ $uri->uidvalidity($r_uidval);
$l_uid //= 0;
- if ($l_uidval != $r_uidval) {
- return "E: $uri UIDVALIDITY mismatch\n".
- "E: local=$l_uidval != remote=$r_uidval";
- }
my $r_uid = $r_uidnext - 1;
- if ($l_uid > $r_uid) {
- return "E: $uri local UID exceeds remote ($l_uid > $r_uid)\n".
- "E: $uri strangely, UIDVALIDLITY matches ($l_uidval)\n";
- }
+ return <<EOF if $l_uid > $r_uid;
+E: $uri local UID exceeds remote ($l_uid > $r_uid)
+E: $uri strangely, UIDVALIDLITY matches ($l_uidval)
+EOF
return if $l_uid >= $r_uid; # nothing to do
$l_uid ||= 1;
my ($mod, $shard) = @{$self->{shard_info} // []};
# I wish "UID FETCH $START:*" could work, but:
# 1) servers do not need to return results in any order
# 2) Mail::IMAPClient doesn't offer a streaming API
- $uids = $mic->search("UID $l_uid:*") or
+ unless ($uids = $mic->search("UID $l_uid:*")) {
+ return if $!{EINTR} && $self->{quit};
return "E: $uri UID SEARCH $l_uid:* error: $!";
+ }
return if scalar(@$uids) == 0;
# RFC 3501 doesn't seem to indicate order of UID SEARCH
local $0 = "UID:$batch $mbx $sec";
my $r = $mic->fetch_hash($batch, $req, 'FLAGS');
unless ($r) { # network error?
+ last if $!{EINTR} && $self->{quit};
$err = "E: $uri UID FETCH $batch error: $!";
last;
}
# messages get deleted, so holes appear
my $per_uid = delete $r->{$uid} // next;
my $raw = delete($per_uid->{$key}) // next;
- _imap_do_msg($self, $uri, $uid, \$raw,
+ _imap_do_msg($self, $$uri, $uid, \$raw,
$per_uid->{FLAGS});
$last_uid = $uid;
last if $self->{quit};
}
last if $self->{quit};
}
+ run_commit_cb($self);
$itrk->update_last($r_uidval, $last_uid) if $itrk;
} until ($err || $self->{quit});
$err;
local $self->{eml_each} = [ $eml_cb, @args ];
$err = _imap_fetch_all($self, $mic, $uri);
} else {
- $err = "E: not connected: $!";
+ $err = "E: <$uri> not connected: $!";
}
warn $err if $err;
$mic;
my $sec = uri_section($uri);
my ($nr, $beg, $end) = $nn->group($group);
unless (defined($nr)) {
- chomp(my $msg = $nn->message);
+ my $msg = ndump($nn->message);
return "E: GROUP $group <$sec> $msg";
}
# IMAPTracker is also used for tracking NNTP, UID == article number
# LIST.ACTIVE can get the equivalent of UIDVALIDITY, but that's
# expensive. So we assume newsgroups don't change:
- my $itrk = $self->{incremental} ?
- PublicInbox::IMAPTracker->new($$uri) : 0;
- my (undef, $l_art) = $itrk ? $itrk->get_last : ();
+ my ($itrk, $l_art) = _itrk_last($self, $uri);
# allow users to specify articles to refetch
# cf. https://tools.ietf.org/id/draft-gilman-news-url-01.txt
return if $l_art >= $end; # nothing to do
$beg = $l_art + 1;
}
- my ($err, $art);
+ my ($err, $art, $last_art, $kw); # kw stays undef, no keywords in NNTP
unless ($self->{quiet}) {
warn "# $uri fetching ARTICLE $beg..$end\n";
}
- my $last_art;
my $n = $self->{max_batch};
for ($beg..$end) {
last if $self->{quit};
$art = $_;
if (--$n < 0) {
+ run_commit_cb($self);
$itrk->update_last(0, $last_art) if $itrk;
$n = $self->{max_batch};
}
my $raw = $nn->article($art);
unless (defined($raw)) {
- my $msg = $nn->message;
+ my $msg = ndump($nn->message);
if ($nn->code == 421) { # pseudo response from Net::Cmd
$err = "E: $msg";
last;
$raw = join('', @$raw);
$raw =~ s/\r\n/\n/sg;
my ($eml_cb, @args) = @{$self->{eml_each}};
- $eml_cb->($uri, $art, [], PublicInbox::Eml->new(\$raw), @args);
+ $eml_cb->($uri, $art, $kw, PublicInbox::Eml->new(\$raw), @args);
$last_art = $art;
}
+ run_commit_cb($self);
$itrk->update_last(0, $last_art) if $itrk;
$err;
}
my $sec = uri_section($uri);
local $0 = $uri->group ." $sec";
my $nn = nn_get($self, $uri);
+ return if $self->{quit};
my $err;
if ($nn) {
local $self->{eml_each} = [ $eml_cb, @args ];
$err = _nntp_fetch_all($self, $nn, $uri);
} else {
- $err = "E: not connected: $!";
+ $err = "E: <$uri> not connected: $!";
}
warn $err if $err;
$nn;