imap: FETCH: proper MSN => UID mapping for requests

This finally seems to make mutt header caching behave properly.

We expect to be able to safely load 50K IV/UVs in memory without
OOM, since that's "only" 1.6 MB that won't live beyond a single
event loop iteration.  So create a simple array which can
quickly map MSNs in requests to UIDs and not leave out messages.

MSNs in the FETCH response will NOT be correct, since it's
inefficient to implement properly and mutt doesn't seem to
care.

Since the conversion code is easily shared, "UID SEARCH" can
allow the same MSN => UID mapping non-UID "FETCH" does.

diff --git a/lib/PublicInbox/IMAP.pm b/lib/PublicInbox/IMAP.pm
index 7efe538764683deaec7400a9b9e2eff892d80a6c..ff01d0b5e8ed6b6317ffa0739ebc9166f806a2dc 100644 (file)
--- a/lib/PublicInbox/IMAP.pm
+++ b/lib/PublicInbox/IMAP.pm
@@ -16,7 +16,12 @@
 #   are for the limitations of git clients, while slices are
 #   for the limitations of IMAP clients.
 #
-# * sequence numbers are estimated based on slice
+# * sequence numbers are estimated based on slice.  If they
+#   wrong, they're higher than than the corresponding UID
+#   because UIDs have gaps due to spam removals.
+#   We only support an ephemeral mapping non-UID "FETCH"
+#   because mutt header caching relies on it; mutt uses
+#   UID requests everywhere else.
 
 package PublicInbox::IMAP;
 use strict;
@@ -957,9 +962,17 @@ sub cmd_uid_fetch ($$$$;@) {
        long_response($self, $cb, $tag, [], $range_info, $ops, $partial);
 }
 
+# returns an arrayref of UIDs, so MSNs can be translated via:
+# $msn2uid->[$MSN-1] => $UID
+sub msn2uid ($) {
+       my ($self) = @_;
+       my $x = $self->{uid_base};
+       $self->{ibx}->over->uid_range($x + 1, $x + UID_SLICE);
+}
+
 sub msn_to_uid_range ($$) {
-       my $uid_base = $_[0]->{uid_base};
-       $_[1] =~ s/([0-9]+)/$uid_base + $1/sge;
+       my $msn2uid = $_[0];
+       $_[1] =~ s!([0-9]+)!$msn2uid->[$1 - 1] // ($msn2uid->[-1] + 1)!sge;
 }
 
 sub cmd_fetch ($$$$;@) {
@@ -970,7 +983,7 @@ sub cmd_fetch ($$$$;@) {
 
        # cb is one of fetch_blob, fetch_smsg, fetch_uid
        $range_csv = 'bad' if $range_csv !~ $valid_range;
-       msn_to_uid_range($self, $range_csv);
+       msn_to_uid_range(msn2uid($self), $range_csv);
        my $range_info = range_step($self, \$range_csv);
        return "$tag $range_info\r\n" if !ref($range_info);
        long_response($self, $cb, $tag, [], $range_info, $ops, $partial);
@@ -1070,13 +1083,14 @@ sub parse_query {
        my $sql = ''; # date conditions, {sql} deleted if Xapian is needed
        my $xap = '';
        my $q = { sql => \$sql, xap => \$xap };
+       my $msn2uid;
        while (@$rest) {
                my $k = uc(shift @$rest);
                # default criteria
                next if $k =~ /\A(?:ALL|RECENT|UNSEEN|NEW)\z/;
                next if $k eq 'AND'; # the default, until we support OR
                if ($k =~ $valid_range) { # convert sequence numbers to UIDs
-                       msn_to_uid_range($self, $k);
+                       msn_to_uid_range($msn2uid //= msn2uid($self), $k);
                        push @{$q->{uid}}, $k;
                } elsif ($k eq 'UID') {
                        $k = shift(@$rest) // '';