X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=t%2Fimapd.t;h=43de867548c651ab89bb98bc51d8211e768cee28;hb=2f8d15b1170b137a0512a4f3163b236a5dcbf0b8;hp=233be9f2c0a796b469d99dcd2ddfbe94deb12692;hpb=b1a86566fa4ffc45a2900f1de43c48c5be76905a;p=public-inbox.git diff --git a/t/imapd.t b/t/imapd.t index 233be9f2..43de8675 100644 --- a/t/imapd.t +++ b/t/imapd.t @@ -1,5 +1,5 @@ #!perl -w -# Copyright (C) 2020 all contributors +# Copyright (C) all contributors # License: AGPL-3.0+ # end-to-end IMAP tests, see unit tests in t/imap.t, too use strict; @@ -7,8 +7,7 @@ use Test::More; use Time::HiRes (); use PublicInbox::TestCommon; use PublicInbox::Config; -use PublicInbox::Spawn qw(which); -require_mods(qw(DBD::SQLite Mail::IMAPClient Mail::IMAPClient::BodyStructure)); +require_mods(qw(-imapd Mail::IMAPClient)); my $imap_client = 'Mail::IMAPClient'; my $can_compress = $imap_client->can('compress'); if ($can_compress) { # hope this gets fixed upstream, soon @@ -19,10 +18,10 @@ if ($can_compress) { # hope this gets fixed upstream, soon require_ok 'PublicInbox::IMAP'; my $first_range = '0'; -my $level = '-Lbasic'; +my $level = 'basic'; SKIP: { require_mods('Search::Xapian', 1); - $level = '-Lmedium'; + $level = 'medium'; }; my @V = (1); @@ -30,42 +29,41 @@ push(@V, 2) if require_git('2.6', 1); my ($tmpdir, $for_destroy) = tmpdir(); my $home = "$tmpdir/home"; -local $ENV{HOME} = $home; - +BAIL_OUT "mkdir: $!" unless (mkdir($home) and mkdir("$home/.public-inbox")); +my @ibx; +open my $cfgfh, '>', "$home/.public-inbox/config" or BAIL_OUT; +print $cfgfh < $addr }; - run_script(['-mda', '--no-precheck'], $env, { 0 => $fh }) or - BAIL_OUT('-mda delivery'); - if ($V == 1) { - run_script(['-index', $inboxdir]) or BAIL_OUT("index $?"); - } + my $ibx = create_inbox("i$V-$level", + tmpdir => "$tmpdir/i$V", version => $V, + indexlevel => $level, sub { + my ($im) = @_; + $im->add($eml //= eml_load('t/utf8.eml')) or BAIL_OUT; + }); + push @ibx, $ibx; + $ibx->{newsgroup} = "inbox.i$V"; + print $cfgfh <{inboxdir} + address = $ibx->{-primary_address}; + newsgroup = inbox.i$V + url = http://example.com/i$V + indexlevel = $level +EOF } +close $cfgfh or BAIL_OUT; +local $ENV{HOME} = $home; my $sock = tcp_server(); my $err = "$tmpdir/stderr.log"; my $out = "$tmpdir/stdout.log"; my $cmd = [ '-imapd', '-W0', "--stdout=$out", "--stderr=$err" ]; my $td = start_script($cmd, undef, { 3 => $sock }) or BAIL_OUT("-imapd: $?"); -my %mic_opt = ( - Server => $sock->sockhost, - Port => $sock->sockport, - Uid => 1, -); +my ($ihost, $iport) = tcp_host_port($sock); +my %mic_opt = ( Server => $ihost, Port => $iport, Uid => 1 ); my $mic = $imap_client->new(%mic_opt); my $pre_login_capa = $mic->capability; is(grep(/\AAUTH=ANONYMOUS\z/, @$pre_login_capa), 1, @@ -86,28 +84,33 @@ my $post_auth_anon_capa = $mic->capability; is_deeply($post_auth_anon_capa, $post_login_capa, 'auth anon has same capabilities'); my $e; +ok($mic->noop, 'NOOP'); +ok($mic->noop, 'NOOP (again)'); # for warnings ok(!$mic->examine('foo') && ($e = $@), 'EXAMINE non-existent'); like($e, qr/\bNO\b/, 'got a NO on EXAMINE for non-existent'); ok(!$mic->select('foo') && ($e = $@), 'EXAMINE non-existent'); like($e, qr/\bNO\b/, 'got a NO on EXAMINE for non-existent'); my $mailbox1 = "inbox.i1.$first_range"; ok($mic->select('inbox.i1'), 'SELECT on parent succeeds'); +ok($mic->noop, 'NOOP while selected'); +ok($mic->noop, 'NOOP again while selected'); # check warnings later ok($mic->select($mailbox1), 'SELECT succeeds'); ok($mic->examine($mailbox1), 'EXAMINE succeeds'); my @raw = $mic->status($mailbox1, qw(Messages uidnext uidvalidity)); is(scalar(@raw), 2, 'got status response'); like($raw[0], qr/\A\*\x20STATUS\x20inbox\.i1\.$first_range\x20 - \(MESSAGES\x20\d+\x20UIDNEXT\x20\d+\x20UIDVALIDITY\x20\d+\)\r\n/sx); + \(MESSAGES\x20[1-9][0-9]*\x20 + UIDNEXT\x20\d+\x20UIDVALIDITY\x20\d+\)\r\n/sx); like($raw[1], qr/\A\S+ OK /, 'finished status response'); my @orig_list = @raw = $mic->list; -like($raw[0], qr/^\* LIST \(.*?\) "\." inbox/, +like($raw[0], qr/^\* LIST \(.*?\) "\." INBOX/, 'got an inbox'); like($raw[-1], qr/^\S+ OK /, 'response ended with OK'); -is(scalar(@raw), scalar(@V) + 4, 'default LIST response'); +is(scalar(@raw), scalar(@V) * 2 + 2, 'default LIST response'); @raw = $mic->list('', 'inbox.i1'); is(scalar(@raw), 2, 'limited LIST response'); -like($raw[0], qr/^\* LIST \(.*?\) "\." inbox/, +like($raw[0], qr/^\* LIST \(.*?\) "\." INBOX/, 'got an inbox.i1'); like($raw[-1], qr/^\S+ OK /, 'response ended with OK'); @@ -119,6 +122,35 @@ $ret = $mic->search('uid 1:1') or BAIL_OUT "SEARCH FAIL $@"; is_deeply($ret, [ 1 ], 'search UID 1:1 works'); $ret = $mic->search('uid 1:*') or BAIL_OUT "SEARCH FAIL $@"; is_deeply($ret, [ 1 ], 'search UID 1:* works'); +$ret = $mic->search('DELETED') or BAIL_OUT "SEARCH FAIL $@"; +is_deeply($ret, [], 'searching for DELETED returns nothing'); + +SKIP: { + skip 'Xapian missing', 8 if $level eq 'basic'; + my $x = $mic->search(qw(smaller 99999)); + is_deeply($x, [1], 'SMALLER works with Xapian (hit)'); + $x = $mic->search(qw(smaller 9)); + is_deeply($x, [], 'SMALLER works with Xapian (miss)'); + + $x = $mic->search(qw(larger 99999)); + is_deeply($x, [], 'LARGER works with Xapian (miss)'); + $x = $mic->search(qw(larger 9)); + is_deeply($x, [1], 'LARGER works with Xapian (hit)'); + + $x = $mic->search(qw(HEADER Message-ID testmessage@example.com)); + is_deeply($x, [1], 'HEADER Message-ID works'); + + $x = $mic->search(qw(DRAFT HEADER Message-ID testmessage@example.com)); + is_deeply($x, [], 'impossible (DRAFT) key filters out match'); + + $x = $mic->search(qw(HEADER Message-ID miss)); + is_deeply($x, [], 'HEADER Message-ID can miss'); + + my @q = qw[OR HEADER Message-ID testmessage@example.com + (OR FROM Ryan (OR TO Joe CC Scott))]; + $x = $mic->search(join(' ', @q)); + is_deeply($x, [1], 'nested query works'); +} is_deeply(scalar $mic->flags('1'), [], '->flags works'); { @@ -129,10 +161,16 @@ is_deeply(scalar $mic->flags('1'), [], '->flags works'); my $exp = $mic->fetch_hash(1, 'UID'); $ret = $mic->fetch_hash('559:*', 'UID'); is_deeply($ret, $exp, 'beginning range too big'); - for my $r (qw(559:558 558:559)) { - $ret = $mic->fetch_hash($r, 'UID'); + { + my @w; # Mail::IMAPClient hits a warning via overload + local $SIG{__WARN__} = sub { push @w, @_ }; + $ret = $mic->fetch_hash(my $r = '559:558', 'UID'); is_deeply($ret, {}, "out-of-range UID FETCH $r"); + @w = grep(!/\boverload\.pm\b/, @w); + is_deeply(\@w, [], 'no unexpected warning'); } + $ret = $mic->fetch_hash(my $r = '558:559', 'UID'); + is_deeply($ret, {}, "out-of-range UID FETCH $r"); } for my $r ('1:*', '1') { @@ -198,8 +236,8 @@ for my $r ('1:*', '1') { is(lc($bs->bodytype), 'text', '->bodytype'); is(lc($bs->bodyenc), '8bit', '->bodyenc'); } - -is_deeply([$mic->has_capability('COMPRESS')], ['DEFLATE'], 'deflate cap'); +ok($mic->has_capability('COMPRESS') || + $mic->has_capability('COMPRESS=DEFLATE'), 'deflate cap'); SKIP: { skip 'Mail::IMAPClient too old for ->compress', 2 if !$can_compress; my $c = $imap_client->new(%mic_opt); @@ -213,10 +251,7 @@ ok($mic->logout, 'logout works'); my $have_inotify = eval { require Linux::Inotify2; 1 }; -my $pi_config = PublicInbox::Config->new; -$pi_config->each_inbox(sub { - my ($ibx) = @_; - my $env = { ORIGINAL_RECIPIENT => $ibx->{-primary_address} }; +for my $ibx (@ibx) { my $name = $ibx->{name}; my $ng = $ibx->{newsgroup}; my $mic = $imap_client->new(%mic_opt); @@ -224,14 +259,13 @@ $pi_config->each_inbox(sub { my $mb = "$ng.$first_range"; my $uidnext = $mic->uidnext($mb); # we'll fetch BODYSTRUCTURE on this ok($uidnext, 'got uidnext for later fetch'); - is_deeply([$mic->has_capability('IDLE')], ['IDLE'], "IDLE capa $name"); + ok($mic->has_capability('IDLE'), "IDLE capa $name"); ok(!$mic->idle, "IDLE fails w/o SELECT/EXAMINE $name"); ok($mic->examine($mb), "EXAMINE $ng succeeds"); ok(my $idle_tag = $mic->idle, "IDLE succeeds on $ng"); - - open(my $fh, '<', 't/data/message_embed.eml') or BAIL_OUT("open: $!"); - run_script(['-mda', '--no-precheck'], $env, { 0 => $fh }) or - BAIL_OUT('-mda delivery'); + my $im = $ibx->importer(0); + $im->add(eml_load 't/data/message_embed.eml') or BAIL_OUT; + $im->done; my $t0 = Time::HiRes::time(); ok(my @res = $mic->idle_data(11), "IDLE succeeds on $ng"); is(grep(/\A\* [0-9] EXISTS\b/, @res), 1, 'got EXISTS message'); @@ -258,31 +292,14 @@ $pi_config->each_inbox(sub { # ensure IDLE persists across HUP, w/o extra watches or FDs $td->kill('HUP') or BAIL_OUT "failed to kill -imapd: $!"; - SKIP: { - skip 'no inotify fdinfo (or support)', 2 if !@ino_info; - my (@tmp, %prev); - local $/ = "\n"; - my $end = time + 5; - until (time > $end) { - select undef, undef, undef, 0.01; - open my $fh, '<', $ino_fdinfo or - BAIL_OUT "$ino_fdinfo: $!"; - %prev = map { $_ => 1 } @ino_info; - @tmp = grep(/^inotify wd:/, <$fh>); - if (scalar(@tmp) == scalar(@ino_info)) { - delete @prev{@tmp}; - last if scalar(keys(%prev)) == @ino_info; - } - } - is(scalar @tmp, scalar @ino_info, - 'old inotify watches replaced'); - is(scalar keys %prev, scalar @ino_info, - 'no previous watches overlap'); - }; + for my $n (1..2) { # kick the event loop so we know HUP is done + my $m = $imap_client->new(%mic_opt); + ok($m->login && $m->IsAuthenticated && $m->logout, + "connection $n works after HUP"); + } - open($fh, '<', 't/data/0001.patch') or BAIL_OUT("open: $!"); - run_script(['-mda', '--no-precheck'], $env, { 0 => $fh }) or - BAIL_OUT('-mda delivery'); + $im->add(eml_load 't/data/0001.patch') or BAIL_OUT; + $im->done; $t0 = Time::HiRes::time(); ok(@res = $mic->idle_data(11), "IDLE succeeds on $ng after HUP"); is(grep(/\A\* [0-9] EXISTS\b/, @res), 1, 'got EXISTS message'); @@ -332,7 +349,12 @@ EOF ok($mic->examine($ng), 'EXAMINE on dummy'); @hits = $mic->search('SENTSINCE' => '18-Apr-2020'); is_deeply(\@hits, [], 'search on dummy with condition works'); -}); # each_inbox + ok(!$mic->search('SENTSINCE' => '18-Abr-2020'), 'bad month fails'); + $mic->Uid(0); + my $ret = $mic->fetch_hash(2, 'RFC822'); + is_deeply($ret, {}, + 'MSN FETCH on empty dummy will not trigger warnings, later'); +}; # for @ibx # message sequence numbers :< is($mic->Uid(0), 0, 'disable UID on '.ref($mic)); @@ -341,12 +363,42 @@ $ret = $mic->fetch_hash('1,2:3', 'RFC822') or BAIL_OUT "FETCH $@"; is(scalar keys %$ret, 3, 'got all 3 messages with comma-separated sequence'); $ret = $mic->fetch_hash('1:*', 'RFC822') or BAIL_OUT "FETCH $@"; is(scalar keys %$ret, 3, 'got all 3 messages'); + +SKIP: { + # do any clients use non-UID IMAP SEARCH? + skip 'Xapian missing', 3 if $level eq 'basic'; + my $x = $mic->search('all'); + is_deeply($x, [1, 2, 3], 'MSN SEARCH works before rm'); + $x = $mic->search(qw(header subject embedded)); + is_deeply($x, [2], 'MSN SEARCH on Subject works before rm'); + $x = $mic->search('FROM scraper@example.com'); + is_deeply($x, [], "MSN SEARCH miss won't trigger warnings"); +} + { my $rdr = { 0 => \($ret->{1}->{RFC822}) }; my $env = { HOME => $ENV{HOME} }; my @cmd = qw(-learn rm --all); run_script(\@cmd, $env, $rdr) or BAIL_OUT('-learn rm'); } + +SKIP: { + # do any clients use non-UID IMAP SEARCH? We only ensure + # MSN "SEARCH" can return a result which can be retrieved + # via MSN "FETCH" + skip 'Xapian missing', 3 if $level eq 'basic'; + my $x = $mic->search(qw(header subject embedded)); + is(scalar(@$x), 1, 'MSN SEARCH on Subject works after rm'); + SKIP: { + skip 'previous test failed', 1 if !scalar(@$x); + $x = $mic->message_string($x->[0]); + is($x, $ret->{2}->{RFC822}, 'message 2 unchanged'); + } + $x = $mic->search(qw(text embedded)); + is(scalar(@$x), 1, 'MSN SEARCH on TEXT works after rm'); +} + +# FIXME? no EXPUNGE response, yet my $r2 = $mic->fetch_hash('1:*', 'BODY.PEEK[]') or BAIL_OUT "FETCH $@"; is(scalar keys %$r2, 2, 'did not get all 3 messages'); is($r2->{2}->{'BODY[]'}, $ret->{2}->{RFC822}, 'message 2 unchanged'); @@ -365,7 +417,9 @@ is($r2->{2}->{'BODY[HEADER.FIELDS (MESSAGE-ID)]'}, pop @new_list; pop @orig_list; # TODO: not sure if sort order matters, imapd_refresh_finalize - # sorts, for now, but maybe clients don't care... + # doesn't sort, hopefully clients don't care... + @new_list = sort @new_list; + @orig_list = sort @orig_list; is_deeply(\@new_list, \@orig_list, 'LIST identical'); } ok($mic->close, 'CLOSE works'); @@ -382,9 +436,107 @@ ok($mic->logout, 'logged out'); like(<$c>, qr/\Atagonly BAD Error in IMAP command/, 'tag-only line'); } +SKIP: { + use_ok 'PublicInbox::InboxIdle'; + require_git('1.8.5', 1) or + skip('git 1.8.5+ needed for --urlmatch', 4); + my $old_env = { HOME => $ENV{HOME} }; + my $home = "$tmpdir/watch_home"; + mkdir $home or BAIL_OUT $!; + mkdir "$home/.public-inbox" or BAIL_OUT $!; + local $ENV{HOME} = $home; + my $name = 'watchimap'; + my $addr = "i1-$level\@example.com"; + my $url = "http://example.com/i1"; + my $inboxdir = "$tmpdir/watchimap"; + my $cmd = ['-init', '-V2', '-Lbasic', $name, $inboxdir, $url, $addr]; + my $imapurl = "imap://$ihost:$iport/inbox.i1.0"; + run_script($cmd) or BAIL_OUT("init $name"); + xsys(qw(git config), "--file=$home/.public-inbox/config", + "publicinbox.$name.watch", + $imapurl) == 0 or BAIL_OUT "git config $?"; + my $cfg = PublicInbox::Config->new; + PublicInbox::DS->Reset; + my $ii = PublicInbox::InboxIdle->new($cfg); + my $cb = sub { PublicInbox::DS->SetPostLoopCallback(sub {}) }; + my $obj = bless \$cb, 'PublicInbox::TestCommon::InboxWakeup'; + $cfg->each_inbox(sub { $_[0]->subscribe_unlock('ident', $obj) }); + my $watcherr = "$tmpdir/watcherr"; + open my $err_wr, '>>', $watcherr or BAIL_OUT $!; + open my $err, '<', $watcherr or BAIL_OUT $!; + my $w = start_script(['-watch'], undef, { 2 => $err_wr }); + + diag 'waiting for initial fetch...'; + PublicInbox::DS::event_loop(); + diag 'inbox unlocked on initial fetch, waiting for IDLE'; + + tick until (grep(/I: \S+ idling/, <$err>)); + open my $fh, '<', 't/iso-2202-jp.eml' or BAIL_OUT $!; + $old_env->{ORIGINAL_RECIPIENT} = $addr; + ok(run_script([qw(-mda --no-precheck)], $old_env, { 0 => $fh }), + 'delivered a message for IDLE to kick -watch') or + diag "mda error \$?=$?"; + diag 'waiting for IMAP IDLE wakeup'; + PublicInbox::DS->SetPostLoopCallback(undef); + PublicInbox::DS::event_loop(); + diag 'inbox unlocked on IDLE wakeup'; + + # try again with polling + xsys(qw(git config), "--file=$home/.public-inbox/config", + 'imap.PollInterval', 0.11) == 0 + or BAIL_OUT "git config $?"; + $w->kill('HUP'); + diag 'waiting for -watch reload + initial fetch'; + tick until (grep(/I: will check/, <$err>)); + + open $fh, '<', 't/psgi_attach.eml' or BAIL_OUT $!; + ok(run_script([qw(-mda --no-precheck)], $old_env, { 0 => $fh }), + 'delivered a message for -watch PollInterval'); + + diag 'waiting for PollInterval wakeup'; + PublicInbox::DS->SetPostLoopCallback(undef); + PublicInbox::DS::event_loop(); + diag 'inbox unlocked (poll)'; + $w->kill; + $w->join; + is($?, 0, 'no error in exited -watch process'); + + $cfg->each_inbox(sub { shift->unsubscribe_unlock('ident') }); + $ii->close; + PublicInbox::DS->Reset; + seek($err, 0, 0); + my @err = grep(!/^(?:I:|#)/, <$err>); + is(@err, 0, 'no warnings/errors from -watch'.join(' ', @err)); + + if ($ENV{TEST_KILL_IMAPD}) { # not sure how reliable this test can be + xsys(qw(git config), "--file=$home/.public-inbox/config", + qw(--unset imap.PollInterval)) == 0 + or BAIL_OUT "git config $?"; + truncate($err_wr, 0) or BAIL_OUT $!; + my @t0 = times; + $w = start_script(['-watch'], undef, { 2 => $err_wr }); + seek($err, 0, 0); + tick until (grep(/I: \S+ idling/, <$err>)); + diag 'killing imapd, waiting for CPU spins'; + my $delay = 0.11; + $td->kill(9); + tick $delay; + $w->kill; + $w->join; + is($?, 0, 'no error in exited -watch process'); + my @t1 = times; + my $c = $t1[2] + $t1[3] - $t0[2] - $t0[3]; + my $thresh = (0.9 * $delay); + diag "c=$c, threshold=$thresh"; + ok($c < $thresh, 'did not burn much CPU'); + is_deeply([grep(/ line \d+$/m, <$err>)], [], + 'no backtraces from errors'); + } +} + $td->kill; $td->join; -is($?, 0, 'no error in exited process'); +is($?, 0, 'no error in exited process') if !$ENV{TEST_KILL_IMAPD}; open my $fh, '<', $err or BAIL_OUT("open $err failed: $!"); my $eout = do { local $/; <$fh> }; unlike($eout, qr/wide/i, 'no Wide character warnings');