2 # Copyright (C) 2020 all contributors <meta@public-inbox.org>
3 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
4 # end-to-end IMAP tests, see unit tests in t/imap.t, too
8 use PublicInbox::TestCommon;
9 use PublicInbox::Config;
10 use PublicInbox::Spawn qw(which);
11 require_mods(qw(DBD::SQLite Mail::IMAPClient Mail::IMAPClient::BodyStructure
12 Email::Address::XS||Mail::Address Parse::RecDescent));
13 my $imap_client = 'Mail::IMAPClient';
14 my $can_compress = $imap_client->can('compress');
15 if ($can_compress) { # hope this gets fixed upstream, soon
16 require PublicInbox::IMAPClient;
17 $imap_client = 'PublicInbox::IMAPClient';
20 require_ok 'PublicInbox::IMAP';
21 my $first_range = '0';
25 require_mods('Search::Xapian', 1);
30 push(@V, 2) if require_git('2.6', 1);
32 my ($tmpdir, $for_destroy) = tmpdir();
33 my $home = "$tmpdir/home";
34 local $ENV{HOME} = $home;
37 my $addr = "i$V\@example.com";
39 my $url = "http://example.com/i$V";
40 my $inboxdir = "$tmpdir/$name";
41 my $folder = "inbox.i$V";
42 my $cmd = ['-init', "-V$V", "-L$level", $name, $inboxdir, $url, $addr];
43 run_script($cmd) or BAIL_OUT("init $name");
44 xsys(qw(git config), "--file=$ENV{HOME}/.public-inbox/config",
45 "publicinbox.$name.newsgroup", $folder) == 0 or
46 BAIL_OUT("setting newsgroup $V");
48 xsys(qw(git config), "--file=$ENV{HOME}/.public-inbox/config",
49 'publicinboxmda.spamcheck', 'none') == 0 or
50 BAIL_OUT("config: $?");
52 open(my $fh, '<', 't/utf8.eml') or BAIL_OUT("open t/utf8.eml: $!");
53 my $env = { ORIGINAL_RECIPIENT => $addr };
54 run_script(['-mda', '--no-precheck'], $env, { 0 => $fh }) or
55 BAIL_OUT('-mda delivery');
57 run_script(['-index', $inboxdir]) or BAIL_OUT("index $?");
60 my $sock = tcp_server();
61 my $err = "$tmpdir/stderr.log";
62 my $out = "$tmpdir/stdout.log";
63 my $cmd = [ '-imapd', '-W0', "--stdout=$out", "--stderr=$err" ];
64 my $td = start_script($cmd, undef, { 3 => $sock }) or BAIL_OUT("-imapd: $?");
66 Server => $sock->sockhost,
67 Port => $sock->sockport,
70 my $mic = $imap_client->new(%mic_opt);
71 my $pre_login_capa = $mic->capability;
72 is(grep(/\AAUTH=ANONYMOUS\z/, @$pre_login_capa), 1,
73 'AUTH=ANONYMOUS advertised pre-login');
75 $mic->User('lorelei');
76 $mic->Password('Hunter2');
77 ok($mic->login && $mic->IsAuthenticated, 'LOGIN works');
78 my $post_login_capa = $mic->capability;
79 ok(join("\n", @$pre_login_capa) ne join("\n", @$post_login_capa),
80 'got different capabilities post-login');
82 $mic_opt{Authmechanism} = 'ANONYMOUS';
83 $mic_opt{Authcallback} = sub { '' };
84 $mic = $imap_client->new(%mic_opt);
85 ok($mic && $mic->login && $mic->IsAuthenticated, 'AUTHENTICATE ANONYMOUS');
86 my $post_auth_anon_capa = $mic->capability;
87 is_deeply($post_auth_anon_capa, $post_login_capa,
88 'auth anon has same capabilities');
90 ok(!$mic->examine('foo') && ($e = $@), 'EXAMINE non-existent');
91 like($e, qr/\bNO\b/, 'got a NO on EXAMINE for non-existent');
92 ok(!$mic->select('foo') && ($e = $@), 'EXAMINE non-existent');
93 like($e, qr/\bNO\b/, 'got a NO on EXAMINE for non-existent');
94 my $mailbox1 = "inbox.i1.$first_range";
95 ok($mic->select('inbox.i1'), 'SELECT on parent succeeds');
96 ok($mic->select($mailbox1), 'SELECT succeeds');
97 ok($mic->examine($mailbox1), 'EXAMINE succeeds');
98 my @raw = $mic->status($mailbox1, qw(Messages uidnext uidvalidity));
99 is(scalar(@raw), 2, 'got status response');
100 like($raw[0], qr/\A\*\x20STATUS\x20inbox\.i1\.$first_range\x20
101 \(MESSAGES\x20\d+\x20UIDNEXT\x20\d+\x20UIDVALIDITY\x20\d+\)\r\n/sx);
102 like($raw[1], qr/\A\S+ OK /, 'finished status response');
104 my @orig_list = @raw = $mic->list;
105 like($raw[0], qr/^\* LIST \(.*?\) "\." INBOX/,
107 like($raw[-1], qr/^\S+ OK /, 'response ended with OK');
108 is(scalar(@raw), scalar(@V) * 2 + 2, 'default LIST response');
109 @raw = $mic->list('', 'inbox.i1');
110 is(scalar(@raw), 2, 'limited LIST response');
111 like($raw[0], qr/^\* LIST \(.*?\) "\." INBOX/,
113 like($raw[-1], qr/^\S+ OK /, 'response ended with OK');
115 my $ret = $mic->search('all') or BAIL_OUT "SEARCH FAIL $@";
116 is_deeply($ret, [ 1 ], 'search all works');
117 $ret = $mic->search('uid 1') or BAIL_OUT "SEARCH FAIL $@";
118 is_deeply($ret, [ 1 ], 'search UID 1 works');
119 $ret = $mic->search('uid 1:1') or BAIL_OUT "SEARCH FAIL $@";
120 is_deeply($ret, [ 1 ], 'search UID 1:1 works');
121 $ret = $mic->search('uid 1:*') or BAIL_OUT "SEARCH FAIL $@";
122 is_deeply($ret, [ 1 ], 'search UID 1:* works');
125 skip 'Xapian missing', 7 if $level eq 'basic';
126 my $x = $mic->search(qw(smaller 99999));
127 is_deeply($x, [1], 'SMALLER works with Xapian (hit)');
128 $x = $mic->search(qw(smaller 9));
129 is_deeply($x, [], 'SMALLER works with Xapian (miss)');
131 $x = $mic->search(qw(larger 99999));
132 is_deeply($x, [], 'LARGER works with Xapian (miss)');
133 $x = $mic->search(qw(larger 9));
134 is_deeply($x, [1], 'LARGER works with Xapian (hit)');
136 $x = $mic->search(qw(HEADER Message-ID testmessage@example.com));
137 is_deeply($x, [1], 'HEADER Message-ID works');
138 $x = $mic->search(qw(HEADER Message-ID miss));
139 is_deeply($x, [], 'HEADER Message-ID can miss');
141 my @q = qw[OR HEADER Message-ID testmessage@example.com
142 (OR FROM Ryan (OR TO Joe CC Scott))];
143 $x = $mic->search(join(' ', @q));
144 is_deeply($x, [1], 'nested query works');
147 is_deeply(scalar $mic->flags('1'), [], '->flags works');
149 # RFC 3501 section 6.4.8 states:
150 # Also note that a UID range of 559:* always includes the
151 # UID of the last message in the mailbox, even if 559 is
152 # higher than any assigned UID value.
153 my $exp = $mic->fetch_hash(1, 'UID');
154 $ret = $mic->fetch_hash('559:*', 'UID');
155 is_deeply($ret, $exp, 'beginning range too big');
157 my @w; # Mail::IMAPClient hits a warning via overload
158 local $SIG{__WARN__} = sub { push @w, @_ };
159 $ret = $mic->fetch_hash(my $r = '559:558', 'UID');
160 is_deeply($ret, {}, "out-of-range UID FETCH $r");
161 @w = grep(!/\boverload\.pm\b/, @w);
162 is_deeply(\@w, [], 'no unexpected warning');
164 $ret = $mic->fetch_hash(my $r = '558:559', 'UID');
165 is_deeply($ret, {}, "out-of-range UID FETCH $r");
168 for my $r ('1:*', '1') {
169 $ret = $mic->fetch_hash($r, 'RFC822') or BAIL_OUT "FETCH $@";
170 is_deeply([keys %$ret], [1]);
171 like($ret->{1}->{RFC822}, qr/\r\n\r\nThis is a test/, 'read full');
173 # ensure Mail::IMAPClient behaves
174 my $str = $mic->message_string($r) or BAIL_OUT "->message_string: $@";
175 is($str, $ret->{1}->{RFC822}, '->message_string works as expected');
177 my $sz = $mic->fetch_hash($r, 'RFC822.size') or BAIL_OUT "FETCH $@";
178 is($sz->{1}->{'RFC822.SIZE'}, length($ret->{1}->{RFC822}),
181 $ret = $mic->fetch_hash($r, 'RFC822.HEADER') or BAIL_OUT "FETCH $@";
182 is_deeply([keys %$ret], [1]);
183 like($ret->{1}->{'RFC822.HEADER'},
184 qr/^Message-ID: <testmessage\@example\.com>/ms, 'read header');
186 $ret = $mic->fetch_hash($r, 'INTERNALDATE') or BAIL_OUT "FETCH $@";
187 is($ret->{1}->{'INTERNALDATE'}, '01-Jan-1970 00:00:00 +0000',
188 'internaldate matches');
189 ok(!$mic->fetch_hash($r, 'INFERNALDATE'), 'bogus attribute fails');
191 my $envelope = $mic->get_envelope($r) or BAIL_OUT("get_envelope: $@");
192 is($envelope->{bcc}, 'NIL', 'empty bcc');
193 is($envelope->{messageid}, '<testmessage@example.com>', 'messageid');
194 is(scalar @{$envelope->{to}}, 1, 'one {to} header');
195 # *sigh* too much to verify...
196 #use Data::Dumper; diag Dumper($envelope);
198 $ret = $mic->fetch_hash($r, 'FLAGS') or BAIL_OUT "FETCH $@";
199 is_deeply($ret->{1}->{FLAGS}, '', 'no flags');
201 $ret = $mic->fetch_hash($r, 'BODY[1]') or BAIL_OUT "FETCH $@";
202 like($ret->{1}->{'BODY[1]'}, qr/\AThis is a test message/, 'BODY[1]');
204 $ret = $mic->fetch_hash($r, 'BODY[1]<1>') or BAIL_OUT "FETCH $@";
205 like($ret->{1}->{'BODY[1]<1>'}, qr/\Ahis is a test message/,
208 $ret = $mic->fetch_hash($r, 'BODY[1]<2.3>') or BAIL_OUT "FETCH $@";
209 is($ret->{1}->{'BODY[1]<2>'}, "is ", 'BODY[1]<2.3>');
210 $ret = $mic->bodypart_string($r, 1, 3, 2) or
211 BAIL_OUT "bodypart_string $@";
212 is($ret, "is ", 'bodypart string');
214 $ret = $mic->fetch_hash($r, 'BODY[HEADER.FIELDS.NOT (Message-ID)]')
215 or BAIL_OUT "FETCH $@";
216 $ret = $ret->{1}->{'BODY[HEADER.FIELDS.NOT (MESSAGE-ID)]'};
217 unlike($ret, qr/message-id/i, 'Message-ID excluded');
218 like($ret, qr/\r\n\r\n\z/s, 'got header end');
220 $ret = $mic->fetch_hash($r, 'BODY[HEADER.FIELDS (Message-ID)]')
221 or BAIL_OUT "FETCH $@";
222 is($ret->{1}->{'BODY[HEADER.FIELDS (MESSAGE-ID)]'},
223 'Message-ID: <testmessage@example.com>'."\r\n\r\n",
224 'got only Message-ID');
226 my $bs = $mic->get_bodystructure($r) or BAIL_OUT("bodystructure: $@");
227 ok($bs, 'got a bodystructure');
228 is(lc($bs->bodytype), 'text', '->bodytype');
229 is(lc($bs->bodyenc), '8bit', '->bodyenc');
231 ok($mic->has_capability('COMPRESS') ||
232 $mic->has_capability('COMPRESS=DEFLATE'), 'deflate cap');
234 skip 'Mail::IMAPClient too old for ->compress', 2 if !$can_compress;
235 my $c = $imap_client->new(%mic_opt);
236 ok($c && $c->compress, 'compress enabled');
237 ok($c->examine($mailbox1), 'EXAMINE succeeds after COMPRESS');
238 $ret = $c->search('uid 1:*') or BAIL_OUT "SEARCH FAIL $@";
239 is_deeply($ret, [ 1 ], 'search UID 1:* works after compression');
242 ok($mic->logout, 'logout works');
244 my $have_inotify = eval { require Linux::Inotify2; 1 };
246 my $pi_config = PublicInbox::Config->new;
247 $pi_config->each_inbox(sub {
249 my $env = { ORIGINAL_RECIPIENT => $ibx->{-primary_address} };
250 my $name = $ibx->{name};
251 my $ng = $ibx->{newsgroup};
252 my $mic = $imap_client->new(%mic_opt);
253 ok($mic && $mic->login && $mic->IsAuthenticated, "authed $name");
254 my $mb = "$ng.$first_range";
255 my $uidnext = $mic->uidnext($mb); # we'll fetch BODYSTRUCTURE on this
256 ok($uidnext, 'got uidnext for later fetch');
257 ok($mic->has_capability('IDLE'), "IDLE capa $name");
258 ok(!$mic->idle, "IDLE fails w/o SELECT/EXAMINE $name");
259 ok($mic->examine($mb), "EXAMINE $ng succeeds");
260 ok(my $idle_tag = $mic->idle, "IDLE succeeds on $ng");
262 open(my $fh, '<', 't/data/message_embed.eml') or BAIL_OUT("open: $!");
263 run_script(['-mda', '--no-precheck'], $env, { 0 => $fh }) or
264 BAIL_OUT('-mda delivery');
265 my $t0 = Time::HiRes::time();
266 ok(my @res = $mic->idle_data(11), "IDLE succeeds on $ng");
267 is(grep(/\A\* [0-9] EXISTS\b/, @res), 1, 'got EXISTS message');
268 ok((Time::HiRes::time() - $t0) < 10, 'IDLE client notified');
270 my (@ino_info, $ino_fdinfo);
272 skip 'no inotify support', 1 unless $have_inotify;
273 skip 'missing /proc/$PID/fd', 1 if !-d "/proc/$td->{pid}/fd";
275 readlink($_) =~ /\binotify\b/
276 } glob("/proc/$td->{pid}/fd/*");
277 is(scalar(@ino), 1, 'only one inotify FD');
278 my $ino_fd = (split('/', $ino[0]))[-1];
279 $ino_fdinfo = "/proc/$td->{pid}/fdinfo/$ino_fd";
280 if (open my $fh, '<', $ino_fdinfo) {
282 @ino_info = grep(/^inotify wd:/, <$fh>);
283 ok(scalar(@ino_info), 'inotify has watches');
285 skip "$ino_fdinfo missing: $!", 1;
289 # ensure IDLE persists across HUP, w/o extra watches or FDs
290 $td->kill('HUP') or BAIL_OUT "failed to kill -imapd: $!";
292 skip 'no inotify fdinfo (or support)', 2 if !@ino_info;
296 until (time > $end) {
297 select undef, undef, undef, 0.01;
298 open my $fh, '<', $ino_fdinfo or
299 BAIL_OUT "$ino_fdinfo: $!";
300 %prev = map { $_ => 1 } @ino_info;
301 @tmp = grep(/^inotify wd:/, <$fh>);
302 if (scalar(@tmp) == scalar(@ino_info)) {
304 last if scalar(keys(%prev)) == @ino_info;
307 is(scalar @tmp, scalar @ino_info,
308 'old inotify watches replaced');
309 is(scalar keys %prev, scalar @ino_info,
310 'no previous watches overlap');
313 open($fh, '<', 't/data/0001.patch') or BAIL_OUT("open: $!");
314 run_script(['-mda', '--no-precheck'], $env, { 0 => $fh }) or
315 BAIL_OUT('-mda delivery');
316 $t0 = Time::HiRes::time();
317 ok(@res = $mic->idle_data(11), "IDLE succeeds on $ng after HUP");
318 is(grep(/\A\* [0-9] EXISTS\b/, @res), 1, 'got EXISTS message');
319 ok((Time::HiRes::time() - $t0) < 10, 'IDLE client notified');
320 ok($mic->done($idle_tag), 'IDLE DONE');
321 my $bs = $mic->get_bodystructure($uidnext);
322 ok($bs, 'BODYSTRUCTURE ok for deeply nested');
323 $ret = $mic->fetch_hash($uidnext, 'BODY') or BAIL_OUT "FETCH $@";
324 ok($ret->{$uidnext}->{BODY}, 'got something in BODY');
326 # this matches dovecot behavior
327 $ret = $mic->fetch_hash($uidnext, 'BODY[1]') or BAIL_OUT "FETCH $@";
328 is($ret->{$uidnext}->{'BODY[1]'},
329 "testing embedded message harder\r\n", 'BODY[1]');
330 $ret = $mic->fetch_hash($uidnext, 'BODY[2]') or BAIL_OUT "FETCH $@";
331 like($ret->{$uidnext}->{'BODY[2]'},
332 qr/\ADate: Sat, 18 Apr 2020 22:20:20 /, 'BODY[2]');
334 $ret = $mic->fetch_hash($uidnext, 'BODY[2.1.1]') or BAIL_OUT "FETCH $@";
335 is($ret->{$uidnext}->{'BODY[2.1.1]'},
336 "testing embedded message\r\n", 'BODY[2.1.1]');
338 $ret = $mic->fetch_hash($uidnext, 'BODY[2.1.2]') or BAIL_OUT "FETCH $@";
339 like($ret->{$uidnext}->{'BODY[2.1.2]'}, qr/\AFrom: /,
340 'BODY[2.1.2] tip matched');
341 like($ret->{$uidnext}->{'BODY[2.1.2]'},
342 # trailing CRLF may vary depending on MIME parser
343 qr/done_testing;(?:\r\n){1,2}\z/,
344 'BODY[2.1.2] tail matched');
346 $ret = $mic->fetch_hash("1:$uidnext", 'BODY[2.HEADER]') or
347 BAIL_OUT "2.HEADER $@";
348 like($ret->{$uidnext}->{'BODY[2.HEADER]'},
349 qr/\ADate: Sat, 18 Apr 2020 22:20:20 /,
350 '2.HEADER of message/rfc822');
352 $ret = $mic->fetch_hash($uidnext, 'BODY[2.MIME]') or
353 BAIL_OUT "2.MIME $@";
354 is($ret->{$uidnext}->{'BODY[2.MIME]'}, <<EOF, 'BODY[2.MIME]');
355 Content-Type: message/rfc822\r
356 Content-Disposition: attachment; filename="embed2x\.eml"\r
360 my @hits = $mic->search('SENTON' => '18-Apr-2020');
361 is_deeply(\@hits, [ $uidnext ], 'search with date condition works');
362 ok($mic->examine($ng), 'EXAMINE on dummy');
363 @hits = $mic->search('SENTSINCE' => '18-Apr-2020');
364 is_deeply(\@hits, [], 'search on dummy with condition works');
365 ok(!$mic->search('SENTSINCE' => '18-Abr-2020'), 'bad month fails');
368 # message sequence numbers :<
369 is($mic->Uid(0), 0, 'disable UID on '.ref($mic));
370 ok($mic->reconnect, 'reconnected');
371 $ret = $mic->fetch_hash('1,2:3', 'RFC822') or BAIL_OUT "FETCH $@";
372 is(scalar keys %$ret, 3, 'got all 3 messages with comma-separated sequence');
373 $ret = $mic->fetch_hash('1:*', 'RFC822') or BAIL_OUT "FETCH $@";
374 is(scalar keys %$ret, 3, 'got all 3 messages');
377 # do any clients use non-UID IMAP SEARCH?
378 skip 'Xapian missing', 2 if $level eq 'basic';
379 my $x = $mic->search('all');
380 is_deeply($x, [1, 2, 3], 'MSN SEARCH works before rm');
381 $x = $mic->search(qw(header subject embedded));
382 is_deeply($x, [2], 'MSN SEARCH on Subject works before rm');
386 my $rdr = { 0 => \($ret->{1}->{RFC822}) };
387 my $env = { HOME => $ENV{HOME} };
388 my @cmd = qw(-learn rm --all);
389 run_script(\@cmd, $env, $rdr) or BAIL_OUT('-learn rm');
393 # do any clients use non-UID IMAP SEARCH? We only ensure
394 # MSN "SEARCH" can return a result which can be retrieved
396 skip 'Xapian missing', 3 if $level eq 'basic';
397 my $x = $mic->search(qw(header subject embedded));
398 is(scalar(@$x), 1, 'MSN SEARCH on Subject works after rm');
399 $x = $mic->message_string($x->[0]);
400 is($x, $ret->{2}->{RFC822}, 'message 2 unchanged');
401 $x = $mic->search(qw(text embedded));
402 is(scalar(@$x), 1, 'MSN SEARCH on TEXT works after rm');
405 # FIXME? no EXPUNGE response, yet
406 my $r2 = $mic->fetch_hash('1:*', 'BODY.PEEK[]') or BAIL_OUT "FETCH $@";
407 is(scalar keys %$r2, 2, 'did not get all 3 messages');
408 is($r2->{2}->{'BODY[]'}, $ret->{2}->{RFC822}, 'message 2 unchanged');
409 is($r2->{3}->{'BODY[]'}, $ret->{3}->{RFC822}, 'message 3 unchanged');
410 $r2 = $mic->fetch_hash(2, 'BODY.PEEK[HEADER.FIELDS (message-id)]')
411 or BAIL_OUT "FETCH $@";
412 is($r2->{2}->{'BODY[HEADER.FIELDS (MESSAGE-ID)]'},
413 'Message-ID: <20200418222508.GA13918@dcvr>'."\r\n\r\n",
414 'BODY.PEEK[HEADER.FIELDS ...] drops .PEEK');
417 my @new_list = $mic->list;
418 # tag differs in [-1]
419 like($orig_list[-1], qr/\A\S+ OK List done\r\n/, 'orig LIST');
420 like($new_list[-1], qr/\A\S+ OK List done\r\n/, 'new LIST');
423 # TODO: not sure if sort order matters, imapd_refresh_finalize
424 # doesn't sort, hopefully clients don't care...
425 @new_list = sort @new_list;
426 @orig_list = sort @orig_list;
427 is_deeply(\@new_list, \@orig_list, 'LIST identical');
429 ok($mic->close, 'CLOSE works');
430 ok(!$mic->close, 'CLOSE not idempotent');
431 ok($mic->logout, 'logged out');
434 my $c = tcp_connect($sock);
436 like(<$c>, qr/\* OK/, 'got a greeting');
438 like(<$c>, qr/\A\* BAD Error in IMAP command/, 'empty line');
439 print $c "tagonly\r\n";
440 like(<$c>, qr/\Atagonly BAD Error in IMAP command/, 'tag-only line');
445 is($?, 0, 'no error in exited process');
446 open my $fh, '<', $err or BAIL_OUT("open $err failed: $!");
447 my $eout = do { local $/; <$fh> };
448 unlike($eout, qr/wide/i, 'no Wide character warnings');
449 unlike($eout, qr/uninitialized/i, 'no uninitialized warnings');