imap: support sequence number FETCH

We'll return dummy messages for now when sequence numbers go
missing, in case clients can't handle missing messages.

diff --git a/lib/PublicInbox/IMAP.pm b/lib/PublicInbox/IMAP.pm
index 673e164621654a0201ba970af8185b730390412c..2aa7ab346c1fb1462cdc3391906d513047e84796 100644 (file)
--- a/lib/PublicInbox/IMAP.pm
+++ b/lib/PublicInbox/IMAP.pm
@@ -348,11 +348,34 @@ sub fetch_body ($;$) {
        join('', @hold);
 }
 
+sub dummy_message ($$) {
+       my ($seqno, $ibx) = @_;
+       my $ret = <<EOF;
+From: nobody\@localhost\r
+To: nobody\@localhost\r
+Date: Thu, 01 Jan 1970 00:00:00 +0000\r
+Message-ID: <dummy-$seqno\@$ibx->{newsgroup}>\r
+Subject: dummy message #$seqno\r
+\r
+You're seeing this message because your IMAP client didn't use UIDs.\r
+The message which used to use this sequence number was likely spam\r
+and removed by the administrator.\r
+EOF
+       \$ret;
+}
+
 sub uid_fetch_cb { # called by git->cat_async
        my ($bref, $oid, $type, $size, $fetch_m_arg) = @_;
        my ($self, undef, $ibx, undef, undef, $msgs, $want) = @$fetch_m_arg;
        my $smsg = shift @$msgs or die 'BUG: no smsg';
-       $smsg->{blob} eq $oid or die "BUG: $smsg->{blob} != $oid";
+       if (!defined($oid)) {
+               # it's possible to have TOCTOU if an admin runs
+               # public-inbox-(edit|purge), just move onto the next message
+               return unless defined $want->{-seqno};
+               $bref = dummy_message($smsg->{num}, $ibx);
+       } else {
+               $smsg->{blob} eq $oid or die "BUG: $smsg->{blob} != $oid";
+       }
        $$bref =~ s/(?<!\r)\n/\r\n/sg; # make strict clients happy
 
        # fixup old bug from import (pre-a0c07cba0e5d8b6a)
@@ -591,19 +614,19 @@ sub partial_emit ($$$) {
        }
 }
 
-sub cmd_uid_fetch ($$$;@) {
-       my ($self, $tag, $range, @want) = @_;
+sub fetch_common ($$$$) {
+       my ($self, $tag, $range, $want) = @_;
        my $ibx = $self->{ibx} or return "$tag BAD No mailbox selected\r\n";
-       if ($want[0] =~ s/\A\(//s) {
-               $want[-1] =~ s/\)\z//s or return "$tag BAD no rparen\r\n";
+       if ($want->[0] =~ s/\A\(//s) {
+               $want->[-1] =~ s/\)\z//s or return "$tag BAD no rparen\r\n";
        }
        my (%partial, %want);
-       while (defined(my $att = shift @want)) {
+       while (defined(my $att = shift @$want)) {
                $att = uc($att);
                my $x = $FETCH_ATT{$att};
                if ($x) {
                        %want = (%want, %$x);
-               } elsif (!partial_prepare(\%partial, \@want, $att)) {
+               } elsif (!partial_prepare(\%partial, $want, $att)) {
                        return "$tag BAD param: $att\r\n";
                }
        }
@@ -629,8 +652,54 @@ sub cmd_uid_fetch ($$$;@) {
        } else {
                return "$tag BAD fetch range\r\n";
        }
-       long_response($self, \&uid_fetch_m, $tag, $ibx,
-                               \$beg, $end, $msgs, \%want);
+       [ $tag, $ibx, \$beg, $end, $msgs, \%want ];
+}
+
+sub cmd_uid_fetch ($$$;@) {
+       my ($self, $tag, $range, @want) = @_;
+       my $args = fetch_common($self, $tag, $range, \@want);
+       ref($args) eq 'ARRAY' ?
+               long_response($self, \&uid_fetch_m, @$args) :
+               $args; # error
+}
+
+sub seq_fetch_m { # long_response
+       my ($self, $tag, $ibx, $beg, $end, $msgs, $want) = @_;
+       if (!@$msgs) { # refill
+               @$msgs = @{$ibx->over->query_xover($$beg, $end)};
+               if (!@$msgs) {
+                       $self->write(\"$tag OK Fetch done\r\n");
+                       return;
+               }
+               $$beg = $msgs->[-1]->{num} + 1;
+       }
+       my $seq = $want->{-seqno}++;
+       my $cur_num = $msgs->[0]->{num};
+       if ($cur_num == $seq) { # as expected
+               my $git = $ibx->git;
+               $git->cat_async_begin; # TODO: actually make async
+               $git->cat_async($msgs->[0]->{blob}, \&uid_fetch_cb, \@_);
+               $git->cat_async_wait;
+       } elsif ($cur_num > $seq) {
+               # send dummy messages until $seq catches up to $cur_num
+               my $smsg = bless { num => $seq, ts => 0 }, 'PublicInbox::Smsg';
+               unshift @$msgs, $smsg;
+               my $bref = dummy_message($seq, $ibx);
+               uid_fetch_cb($bref, undef, undef, undef, \@_);
+       } else { # should not happen
+               die "BUG: cur_num=$cur_num < seq=$seq";
+       }
+       1; # more messages on the way
+}
+
+sub cmd_fetch ($$$;@) {
+       my ($self, $tag, $range, @want) = @_;
+       my $args = fetch_common($self, $tag, $range, \@want);
+       ref($args) eq 'ARRAY' ? do {
+               my $want = $args->[-1];
+               $want->{-seqno} = ${$args->[2]}; # $$beg
+               long_response($self, \&seq_fetch_m, @$args)
+       } : $args; # error
 }
 
 sub uid_search_all { # long_response

diff --git a/t/imapd.t b/t/imapd.t
index fcbbdc09d316d719755ab4b517a95d7ef3be99e3..fc90948e1829c0b67ab1ffea74923e72ea914b92 100644 (file)
--- a/t/imapd.t
+++ b/t/imapd.t
@@ -306,6 +306,24 @@ Content-Disposition: attachment; filename="embed2x\.eml"\r
 EOF
 }); # each_inbox
 
+# message sequence numbers :<
+is($mic->Uid(0), 0, 'disable UID on '.ref($mic));
+ok($mic->reconnect, 'reconnected');
+$ret = $mic->fetch_hash('1:*', 'RFC822') or BAIL_OUT "FETCH $@";
+is(scalar keys %$ret, 3, 'got all 3 messages');
+{
+       my $rdr = { 0 => \($ret->{1}->{RFC822}) };
+       my $env = { HOME => $ENV{HOME} };
+       my @cmd = qw(-learn rm --all);
+       run_script(\@cmd, $env, $rdr) or BAIL_OUT('-learn rm');
+}
+my $r2 = $mic->fetch_hash('1:*', 'RFC822') or BAIL_OUT "FETCH $@";
+is(scalar keys %$r2, 3, 'still got all 3 messages');
+like($r2->{1}->{RFC822}, qr/dummy message #1/, 'got dummy message 1');
+is($r2->{2}->{RFC822}, $ret->{2}->{RFC822}, 'message 2 unchanged');
+is($r2->{3}->{RFC822}, $ret->{3}->{RFC822}, 'message 3 unchanged');
+ok($mic->logout, 'logged out');
+
 $td->kill;
 $td->join;
 is($?, 0, 'no error in exited process');