lei import: --incremental default for NNTP and IMAP

[public-inbox.git] / lib / PublicInbox / NetReader.pm

diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index f5f7100581916b6a6d9380c6ed60ad648b77ee27..c7b43f014f6bd31d5c2e435f8ef3b71640cbfa17 100644 (file)
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -7,11 +7,16 @@ use strict;
 use v5.10.1;
 use parent qw(Exporter PublicInbox::IPC);
 use PublicInbox::Eml;
-
 our %IMAPflags2kw = map {; "\\\u$_" => $_ } qw(seen answered flagged draft);
+$IMAPflags2kw{'$Forwarded'} = 'forwarded';  # RFC 5550
 
 our @EXPORT = qw(uri_section imap_uri nntp_uri);
 
+sub ndump {
+       require Data::Dumper;
+       Data::Dumper->new(\@_)->Useqq(1)->Terse(1)->Dump;
+}
+
 # returns the git config section name, e.g [imap "imaps://user@example.com"]
 # without the mailbox, so we can share connections between different inboxes
 sub uri_section ($) {
@@ -260,10 +265,11 @@ sub imap_common_init ($;$) {
        my $mics = {}; # schema://authority => IMAPClient obj
        for my $uri (@{$self->{imap_order}}) {
                my $sec = uri_section($uri);
-               $mics->{$sec} //= mic_for($self, "$sec/", $mic_args, $lei);
+               my $mic = $mics->{$sec} //=
+                               mic_for($self, "$sec/", $mic_args, $lei) //
+                               die "Unable to continue\n";
                next unless $self->isa('PublicInbox::NetWriter');
                my $dst = $uri->mailbox // next;
-               my $mic = $mics->{$sec};
                next if $mic->exists($dst); # already exists
                $mic->create($dst) or die "CREATE $dst failed <$uri>: $@";
        }
@@ -349,6 +355,8 @@ sub _imap_do_msg ($$$$$) {
                if (my $k = $IMAPflags2kw{$f}) {
                        push @$kw, $k;
                } elsif ($f eq "\\Recent") { # not in JMAP
+               } elsif ($f eq "\\Deleted") { # not in JMAP
+                       return;
                } elsif ($self->{verbose}) {
                        warn "# unknown IMAP flag $f <$uri;uid=$uid>\n";
                }
@@ -365,6 +373,13 @@ sub run_commit_cb ($) {
        $cb->(@args);
 }
 
+sub _itrk ($$) {
+       my ($self, $uri) = @_;
+       return unless $self->{incremental};
+       # itrk_fn is set by lei
+       PublicInbox::IMAPTracker->new($$uri, $self->{itrk_fn});
+}
+
 sub _imap_fetch_all ($$$) {
        my ($self, $mic, $uri) = @_;
        my $sec = uri_section($uri);
@@ -381,8 +396,7 @@ sub _imap_fetch_all ($$$) {
                return "E: $uri cannot get UIDVALIDITY";
        $r_uidnext //= $mic->uidnext($mbx) //
                return "E: $uri cannot get UIDNEXT";
-       my $itrk = $self->{incremental} ?
-                       PublicInbox::IMAPTracker->new($$uri) : 0;
+       my $itrk = _itrk($self, $uri);
        my ($l_uidval, $l_uid) = $itrk ? $itrk->get_last : ();
        $l_uidval //= $r_uidval; # first time
        $l_uid //= 0;
@@ -528,15 +542,14 @@ sub _nntp_fetch_all ($$$) {
        my $sec = uri_section($uri);
        my ($nr, $beg, $end) = $nn->group($group);
        unless (defined($nr)) {
-               chomp(my $msg = $nn->message);
+               my $msg = ndump($nn->message);
                return "E: GROUP $group <$sec> $msg";
        }
 
        # IMAPTracker is also used for tracking NNTP, UID == article number
        # LIST.ACTIVE can get the equivalent of UIDVALIDITY, but that's
        # expensive.  So we assume newsgroups don't change:
-       my $itrk = $self->{incremental} ?
-                       PublicInbox::IMAPTracker->new($$uri) : 0;
+       my $itrk = _itrk($self, $uri);
        my (undef, $l_art) = $itrk ? $itrk->get_last : ();
 
        # allow users to specify articles to refetch
@@ -548,11 +561,10 @@ sub _nntp_fetch_all ($$$) {
                return if $l_art >= $end; # nothing to do
                $beg = $l_art + 1;
        }
-       my ($err, $art);
+       my ($err, $art, $last_art, $kw); # kw stays undef, no keywords in NNTP
        unless ($self->{quiet}) {
                warn "# $uri fetching ARTICLE $beg..$end\n";
        }
-       my $last_art;
        my $n = $self->{max_batch};
        for ($beg..$end) {
                last if $self->{quit};
@@ -564,7 +576,7 @@ sub _nntp_fetch_all ($$$) {
                }
                my $raw = $nn->article($art);
                unless (defined($raw)) {
-                       my $msg = $nn->message;
+                       my $msg = ndump($nn->message);
                        if ($nn->code == 421) { # pseudo response from Net::Cmd
                                $err = "E: $msg";
                                last;
@@ -576,7 +588,7 @@ sub _nntp_fetch_all ($$$) {
                $raw = join('', @$raw);
                $raw =~ s/\r\n/\n/sg;
                my ($eml_cb, @args) = @{$self->{eml_each}};
-               $eml_cb->($uri, $art, [], PublicInbox::Eml->new(\$raw), @args);
+               $eml_cb->($uri, $art, $kw, PublicInbox::Eml->new(\$raw), @args);
                $last_art = $art;
        }
        run_commit_cb($self);