net_writer: start implementing IMAP write support

Requiring TEST_IMAP_WRITE_URL to be set to a writable IMAP
server URL isn't ideal, but it works for now until we have time
to setup a mock dovecot/cyrus/etc... instance for testing.

diff --git a/MANIFEST b/MANIFEST
index 3d9ad616cd488b9daaa240c8bcdbc0a09f90863f..21e37678292496dfd9a481848ba05878c99a5b5b 100644 (file)
--- a/MANIFEST
+++ b/MANIFEST
@@ -213,6 +213,7 @@ lib/PublicInbox/NNTP.pm
 lib/PublicInbox/NNTPD.pm
 lib/PublicInbox/NNTPdeflate.pm
 lib/PublicInbox/NetReader.pm
+lib/PublicInbox/NetWriter.pm
 lib/PublicInbox/NewsWWW.pm
 lib/PublicInbox/OnDestroy.pm
 lib/PublicInbox/Over.pm
@@ -471,6 +472,7 @@ xt/lei-sigpipe.t
 xt/mem-imapd-tls.t
 xt/mem-msgview.t
 xt/msgtime_cmp.t
+xt/net_writer-imap.t
 xt/nntpd-validate.t
 xt/perf-msgview.t
 xt/perf-nntpd.t

diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index 22ba4be74b813fbdffd0142c75e41d234b1fc5e3..92d004bc86df231e33c854dd4b0e12970e64df75 100644 (file)
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -8,6 +8,8 @@ use v5.10.1;
 use parent qw(Exporter PublicInbox::IPC);
 use PublicInbox::Eml;
 
+our %IMAPflags2kw = map {; "\\\u$_" => $_ } qw(seen answered flagged draft);
+
 # TODO: trim this down, this is huge
 our @EXPORT = qw(uri_new uri_scheme uri_section
                mic_for nn_new nn_for
@@ -33,6 +35,7 @@ sub uri_section ($) {
 
 sub auth_anon_cb { '' }; # for Mail::IMAPClient::Authcallback
 
+# mic_for may prompt the user and store auth info, prepares mic_get
 sub mic_for { # mic = Mail::IMAPClient
        my ($self, $url, $mic_args, $lei) = @_;
        require PublicInbox::URIimap;
@@ -286,7 +289,12 @@ sub imap_common_init ($;$) {
        for my $url (@{$self->{imap_order}}) {
                my $uri = PublicInbox::URIimap->new($url);
                my $sec = uri_section($uri);
-               $mics->{$sec} //= mic_for($self, $url, $mic_args, $lei);
+               $mics->{$sec} //= mic_for($self, "$sec/", $mic_args, $lei);
+               next unless $self->isa('PublicInbox::NetWriter');
+               my $dst = $uri->mailbox // next;
+               my $mic = $mics->{$sec};
+               next if $mic->exists($dst); # already exists
+               $mic->create($dst) or die "CREATE $dst failed <$url>: $@";
        }
        $mics;
 }
@@ -312,13 +320,6 @@ sub errors {
        undef;
 }
 
-my %IMAPflags2kw = (
-       '\Seen' => 'seen',
-       '\Answered' => 'answered',
-       '\Flagged' => 'flagged',
-       '\Draft' => 'draft',
-);
-
 sub _imap_do_msg ($$$$$) {
        my ($self, $url, $uid, $raw, $flags) = @_;
        # our target audience expects LF-only, save storage
@@ -418,25 +419,34 @@ sub _imap_fetch_all ($$$) {
        $err;
 }
 
+# uses cached auth info prepared by mic_for
+sub mic_get {
+       my ($self, $sec) = @_;
+       my $mic_arg = $self->{mic_arg}->{$sec} or
+                       die "BUG: no Mail::IMAPClient->new arg for $sec";
+       if (defined(my $cb_name = $mic_arg->{Authcallback})) {
+               if (ref($cb_name) ne 'CODE') {
+                       $mic_arg->{Authcallback} = $self->can($cb_name);
+               }
+       }
+       my $mic = PublicInbox::IMAPClient->new(%$mic_arg);
+       $mic && $mic->IsConnected ? $mic : undef;
+}
+
 sub imap_each {
        my ($self, $url, $eml_cb, @args) = @_;
        my $uri = PublicInbox::URIimap->new($url);
        my $sec = uri_section($uri);
-       my $mic_arg = $self->{mic_arg}->{$sec} or
-                       die "BUG: no Mail::IMAPClient->new arg for $sec";
        local $0 = $uri->mailbox." $sec";
-       my $cb_name = $mic_arg->{Authcallback};
-       if (ref($cb_name) ne 'CODE') {
-               $mic_arg->{Authcallback} = $self->can($cb_name);
-       }
-       my $mic = PublicInbox::IMAPClient->new(%$mic_arg, Debug => 0);
+       my $mic = mic_get($self, $sec);
        my $err;
-       if ($mic && $mic->IsConnected) {
+       if ($mic) {
                local $self->{eml_each} = [ $eml_cb, @args ];
                $err = _imap_fetch_all($self, $mic, $url);
        } else {
                $err = "E: not connected: $!";
        }
+       warn $err if $err;
        $mic;
 }
 

diff --git a/lib/PublicInbox/NetWriter.pm b/lib/PublicInbox/NetWriter.pm
new file mode 100644 (file)
index 0000000..6f0a0b9
--- /dev/null
+++ b/lib/PublicInbox/NetWriter.pm
@@ -0,0 +1,26 @@
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# common writer code for IMAP (and later, JMAP)
+package PublicInbox::NetWriter;
+use strict;
+use v5.10.1;
+use parent qw(PublicInbox::NetReader);
+use PublicInbox::Smsg;
+use PublicInbox::MsgTime qw(msg_timestamp);
+
+my %IMAPkw2flags;
+@IMAPkw2flags{values %PublicInbox::NetReader::IMAPflags2kw} =
+                               keys %PublicInbox::NetReader::IMAPflags2kw;
+
+sub imap_append {
+       my ($mic, $folder, $bref, $smsg, $eml) = @_;
+       $bref //= \($eml->as_string);
+       $smsg //= bless { }, 'PublicInbox::Smsg';
+       $smsg->{ts} //= msg_timestamp($eml // PublicInbox::Eml->new($$bref));
+       my @f = map { $IMAPkw2flags{$_} } @{$smsg->{kw}};
+       $mic->append_string($folder, $$bref, "@f", $smsg->internaldate) or
+               die "APPEND $folder: $@";
+}
+
+1;

diff --git a/xt/net_writer-imap.t b/xt/net_writer-imap.t
new file mode 100644 (file)
index 0000000..ea812f1
--- /dev/null
+++ b/xt/net_writer-imap.t
@@ -0,0 +1,48 @@
+#!perl -w
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict; use v5.10.1; use PublicInbox::TestCommon;
+use Sys::Hostname qw(hostname);
+use POSIX qw(strftime);
+use PublicInbox::OnDestroy;
+use PublicInbox::URIimap;
+use PublicInbox::Config;
+my $imap_url = $ENV{TEST_IMAP_WRITE_URL} or
+       plan skip_all => 'TEST_IMAP_WRITE_URL unset';
+my $uri = PublicInbox::URIimap->new($imap_url);
+defined($uri->path) and
+       plan skip_all => "$imap_url should not be a mailbox (just host:port)";
+require_mods('Mail::IMAPClient');
+require_ok 'PublicInbox::NetWriter';
+my $host = (split(/\./, hostname))[0];
+my ($base) = ($0 =~ m!\b([^/]+)\.[^\.]+\z!);
+my $folder = "INBOX.$base-$host-".strftime('%Y%m%d%H%M%S', gmtime(time)).
+               "-$$-".sprintf('%x', int(rand(0xffffffff)));
+my $nwr = PublicInbox::NetWriter->new;
+$imap_url .= '/' unless substr($imap_url, -1) eq '/';
+my $folder_uri = PublicInbox::URIimap->new("$imap_url/$folder");
+is($folder_uri->mailbox, $folder, 'folder correct') or
+               BAIL_OUT "BUG: bad $$uri";
+$nwr->add_url($$folder_uri);
+is($nwr->errors, undef, 'no errors');
+$nwr->{pi_cfg} = bless {}, 'PublicInbox::Config';
+my $mics = $nwr->imap_common_init;
+my $mic = (values %$mics)[0];
+my $cleanup = PublicInbox::OnDestroy->new(sub {
+       $mic->delete($folder) or fail "delete $folder <$$folder_uri>: $@";
+});
+my $imap_append = $nwr->can('imap_append');
+my $smsg = bless { kw => [ 'seen' ] }, 'PublicInbox::Smsg';
+$imap_append->($mic, $folder, undef, $smsg, eml_load('t/plack-qp.eml'));
+my @res;
+$nwr->{quiet} = 1;
+$nwr->imap_each($$folder_uri, sub {
+       my ($u, $uid, $kw, $eml, $arg) = @_;
+       push @res, [ $kw, $eml ];
+});
+is(scalar(@res), 1, 'got appended message');
+is_deeply(\@res, [ [ [ 'seen' ], eml_load('t/plack-qp.eml') ] ],
+       'uploaded message read back');
+
+undef $cleanup;
+done_testing;