# these values area also used for sorting
sub NEED_SMSG () { 1 }
sub NEED_BLOB () { NEED_SMSG|2 }
-sub NEED_EML () { NEED_BLOB|4 }
-my $OP_EML_NEW = [ NEED_EML - 1, \&op_eml_new ];
+sub CRLF_BREF () { 4 }
+sub EML_HDR () { 8 }
+sub CRLF_HDR () { 16 }
+sub EML_BDY () { 32 }
+sub CRLF_BDY () { 64 }
+my $OP_EML_NEW = [ EML_HDR - 1, \&op_eml_new ];
+my $OP_CRLF_BREF = [ CRLF_BREF, \&op_crlf_bref ];
+my $OP_CRLF_HDR = [ CRLF_HDR, \&op_crlf_hdr ];
+my $OP_CRLF_BDY = [ CRLF_BDY, \&op_crlf_bdy ];
my %FETCH_NEED = (
- 'BODY[HEADER]' => [ NEED_EML, \&emit_rfc822_header ],
- 'BODY[TEXT]' => [ NEED_EML, \&emit_rfc822_text ],
- 'BODY[]' => [ NEED_BLOB, \&emit_rfc822 ],
- 'RFC822.HEADER' => [ NEED_EML, \&emit_rfc822_header ],
- 'RFC822.TEXT' => [ NEED_EML, \&emit_rfc822_text ],
+ 'BODY[HEADER]' => [ NEED_BLOB|EML_HDR|CRLF_HDR, \&emit_rfc822_header ],
+ 'BODY[TEXT]' => [ NEED_BLOB|EML_BDY|CRLF_BDY, \&emit_rfc822_text ],
+ 'BODY[]' => [ NEED_BLOB|CRLF_BREF, \&emit_rfc822 ],
+ 'RFC822.HEADER' => [ NEED_BLOB|EML_HDR|CRLF_HDR, \&emit_rfc822_header ],
+ 'RFC822.TEXT' => [ NEED_BLOB|EML_BDY|CRLF_BDY, \&emit_rfc822_text ],
'RFC822.SIZE' => [ NEED_SMSG, \&emit_rfc822_size ],
- RFC822 => [ NEED_BLOB, \&emit_rfc822 ],
- BODY => [ NEED_EML, \&emit_body ],
- BODYSTRUCTURE => [ NEED_EML, \&emit_bodystructure ],
- ENVELOPE => [ NEED_EML, \&emit_envelope ],
+ RFC822 => [ NEED_BLOB|CRLF_BREF, \&emit_rfc822 ],
+ BODY => [ NEED_BLOB|EML_HDR|EML_BDY, \&emit_body ],
+ BODYSTRUCTURE => [ NEED_BLOB|EML_HDR|EML_BDY, \&emit_bodystructure ],
+ ENVELOPE => [ NEED_BLOB|EML_HDR, \&emit_envelope ],
FLAGS => [ 0, \&emit_flags ],
INTERNALDATE => [ NEED_SMSG, \&emit_internaldate ],
);
# called by PublicInbox::InboxIdle
sub on_inbox_unlock {
my ($self, $ibx) = @_;
- my $new = $ibx->mm->max;
+ my $new = $ibx->over->max;
my $uid_base = $self->{uid_base} // 0;
my $uid_end = $uid_base + UID_BLOCK;
defined(my $old = $self->{-idle_max}) or die 'BUG: -idle_max unset';
# IDLE seems allowed by dovecot w/o a mailbox selected *shrug*
my $ibx = $self->{ibx} or return "$tag BAD no mailbox selected\r\n";
$self->{-idle_tag} = $tag;
- my $max = $ibx->mm->max // 0;
- my $uid_end = ($self->{uid_base} // 0) + UID_BLOCK;
+ my $max = $ibx->over->max;
+ my $uid_end = $self->{uid_base} + UID_BLOCK;
my $sock = $self->{sock} or return;
my $fd = fileno($sock);
# only do inotify on most recent slice
sub inbox_lookup ($$) {
my ($self, $mailbox) = @_;
- my ($ibx, $exists, $uidnext);
+ my ($ibx, $exists, $uidnext, $uid_base);
if ($mailbox =~ /\A(.+)\.([0-9]+)\z/) {
# old mail: inbox.comp.foo.$uid_block_idx
- my ($mb_top, $uid_base) = ($1, $2 * UID_BLOCK);
-
+ my $mb_top = $1;
+ $uid_base = $2 * UID_BLOCK;
$ibx = $self->{imapd}->{mailboxes}->{lc $mailbox} or return;
- $exists = $ibx->mm->max // 0;
- $self->{uid_base} = $uid_base;
- ensure_ranges_exist($self->{imapd}, $ibx, $exists);
- my $uid_end = $uid_base + UID_BLOCK;
- $exists = $uid_end if $exists > $uid_end;
- $uidnext = $exists + 1;
- $exists -= $uid_base;
+ my $max;
+ ($exists, $uidnext, $max) = $ibx->over->imap_status($uid_base,
+ $uid_base + UID_BLOCK);
+ ensure_ranges_exist($self->{imapd}, $ibx, $max);
} else { # check for dummy inboxes
- $ibx = $self->{imapd}->{mailboxes}->{lc $mailbox} or return;
- delete $self->{uid_base};
- $exists = 0;
+ $mailbox = lc $mailbox;
+ $ibx = $self->{imapd}->{mailboxes}->{$mailbox} or return;
+
+ # if "INBOX.foo.bar" is selected and "INBOX.foo.bar.0",
+ # check for new UID ranges (e.g. "INBOX.foo.bar.1")
+ if (my $z = $self->{imapd}->{mailboxes}->{"$mailbox.0"}) {
+ ensure_ranges_exist($self->{imapd}, $z, $z->over->max);
+ }
+
+ $uid_base = $exists = 0;
$uidnext = 1;
}
- ($ibx, $exists, $uidnext);
+ ($ibx, $exists, $uidnext, $uid_base);
}
sub cmd_examine ($$$) {
my ($self, $tag, $mailbox) = @_;
- my ($ibx, $exists, $uidnext) = inbox_lookup($self, $mailbox);
+ my ($ibx, $exists, $uidnext, $base) = inbox_lookup($self, $mailbox);
return "$tag NO Mailbox doesn't exist: $mailbox\r\n" if !$ibx;
+ $self->{uid_base} = $base;
# XXX: do we need this? RFC 5162/7162
my $ret = $self->{ibx} ? "* OK [CLOSED] previous closed\r\n" : '';
$self->requeue if $new_size == 1;
}
-# my ($uid_base, $UID) = @_;
-sub fetch_msn_uid ($$) { '* '.($_[1] - $_[0]).' FETCH (UID '.$_[1] }
+# my ($msn, $UID) = @_;
+sub fetch_msn_uid ($$) { '* '.(${$_[0]}++).' FETCH (UID '.$_[1] }
sub fetch_run_ops {
- my ($self, $uid_base, $smsg, $bref, $ops, $partial) = @_;
- $self->msg_more(fetch_msn_uid($uid_base, $smsg->{num}));
+ my ($self, $msn, $smsg, $bref, $ops, $partial) = @_;
+ $self->msg_more(fetch_msn_uid($msn, $smsg->{num}));
my ($eml, $k);
for (my $i = 0; $i < @$ops;) {
$k = $ops->[$i++];
sub fetch_blob_cb { # called by git->cat_async via git_async_cat
my ($bref, $oid, $type, $size, $fetch_arg) = @_;
- my ($self, undef, $msgs, undef, $ops, $partial) = @$fetch_arg;
+ my ($self, undef, $msgs, $range_info, $ops, $partial) = @$fetch_arg;
my $smsg = shift @$msgs or die 'BUG: no smsg';
if (!defined($oid)) {
# it's possible to have TOCTOU if an admin runs
} else {
$smsg->{blob} eq $oid or die "BUG: $smsg->{blob} != $oid";
}
- $$bref =~ s/(?<!\r)\n/\r\n/sg; # make strict clients happy
-
- # fixup old bug from import (pre-a0c07cba0e5d8b6a)
- $$bref =~ s/\A[\r\n]*From [^\r\n]*\r\n//s;
- fetch_run_ops($self, $self->{uid_base} // 0,
- $smsg, $bref, $ops, $partial);
+ fetch_run_ops($self, $range_info->[3], $smsg, $bref, $ops, $partial);
requeue_once($self);
}
# set $eml once ($_[4] == $eml, $_[3] == $bref)
sub op_eml_new { $_[4] = PublicInbox::Eml->new($_[3]) }
+# s/From / fixes old bug from import (pre-a0c07cba0e5d8b6a)
+sub to_crlf_full {
+ ${$_[0]} =~ s/(?<!\r)\n/\r\n/sg;
+ ${$_[0]} =~ s/\A[\r\n]*From [^\r\n]*\r\n//s;
+}
+
+sub op_crlf_bref { to_crlf_full($_[3]) }
+
+sub op_crlf_hdr { to_crlf_full($_[4]->{hdr}) }
+
+sub op_crlf_bdy { ${$_[4]->{bdy}} =~ s/(?<!\r)\n/\r\n/sg if $_[4]->{bdy} }
+
sub uid_clamp ($$$) {
my ($self, $beg, $end) = @_;
- my $uid_min = ($self->{uid_base} // 0) + 1;
+ my $uid_min = $self->{uid_base} + 1;
my $uid_end = $uid_min + UID_BLOCK - 1;
$$beg = $uid_min if $$beg < $uid_min;
$$end = $uid_end if $$end > $uid_end;
($beg, $end) = ($1 + 0, $2 + 0);
} elsif ($range =~ /\A([0-9]+):\*\z/) {
$beg = $1 + 0;
- $end = $self->{ibx}->mm->max // 0;
+ $end = $self->{ibx}->over->max;
my $uid_end = $self->{uid_base} + UID_BLOCK;
$end = $uid_end if $end > $uid_end;
$beg = $end if $beg > $end;
return 'BAD fetch range';
}
uid_clamp($self, \$beg, \$end) if defined($range);
- [ $beg, $end, $$range_csv ];
+ my $msn = $beg - $self->{uid_base};
+ [ $beg, $end, $$range_csv, \$msn ];
}
sub refill_range ($$$) {
return;
}
}
- my $uid_base = $self->{uid_base} // 0;
- fetch_run_ops($self, $uid_base, $_, undef, $ops) for @$msgs;
+ fetch_run_ops($self, $range_info->[3], $_, undef, $ops) for @$msgs;
@$msgs = ();
1; # more
}
}
# continue looping
}
- my $uid_base = $self->{uid_base} // 0;
my ($i, $k);
+ my $msn = $range_info->[3];
for (@$uids) {
- $self->msg_more(fetch_msn_uid($uid_base, $_));
+ $self->msg_more(fetch_msn_uid($msn, $_));
for ($i = 0; $i < @$ops;) {
$k = $ops->[$i++];
$ops->[$i++]->($self, $k);
}
my $str = $eml->header_obj->as_string;
$str =~ s/$hdrs_re//g;
+ $str =~ s/(?<!\r)\n/\r\n/sg;
$str .= "\r\n";
}
$eml = eml_body_idx($eml, $section_idx) or return;
}
my $str = $eml->header_obj->as_string;
- join('', ($str =~ m/($hdrs_re)/g), "\r\n");
+ $str = join('', ($str =~ m/($hdrs_re)/g));
+ $str =~ s/(?<!\r)\n/\r\n/sg;
+ $str .= "\r\n";
}
-sub partial_prepare ($$$) {
- my ($partial, $want, $att) = @_;
+sub partial_prepare ($$$$) {
+ my ($need, $partial, $want, $att) = @_;
# recombine [ "BODY[1.HEADER.FIELDS", "(foo", "bar)]" ]
# back to: "BODY[1.HEADER.FIELDS (foo bar)]"
(?:\.(HEADER|MIME|TEXT))? # 2 - section_name
\](?:<([0-9]+)(?:\.([0-9]+))?>)?\z/sx) { # 3, 4
$partial->{$att} = [ \&partial_body, $1, $2, $3, $4 ];
+ $$need |= CRLF_BREF|EML_HDR|EML_BDY;
} elsif ($att =~ /\ABODY\[(?:([0-9]+(?:\.[0-9]+)*)\.)? # 1 - section_idx
(?:HEADER\.FIELDS(\.NOT)?)\x20 # 2
\(([A-Z0-9\-\x20]+)\) # 3 - hdrs
: \&partial_hdr_get,
$1, undef, $4, $5 ];
$tmp->[2] = hdrs_regexp($3);
+
+ # don't emit CRLF_HDR instruction, here, partial_hdr_*
+ # will do CRLF conversion with only the extracted result
+ # and not waste time converting lines we don't care about.
+ $$need |= EML_HDR;
} else {
undef;
}
while (my ($k, $fl_cb) = each %$x) {
next if $seen{$k}++;
$need |= $fl_cb->[0];
-
- # insert a special op to convert $bref to $eml
- # the first time we need it
- if ($need == NEED_EML && !$seen{$need}++) {
- push @op, $OP_EML_NEW;
- }
- # $fl_cb = [ flags, \&emit_foo ]
- push @op, [ @$fl_cb , $k ];
+ push @op, [ @$fl_cb, $k ];
}
- } elsif (!partial_prepare(\%partial, $want, $att)) {
+ } elsif (!partial_prepare(\$need, \%partial, $want, $att)) {
return "BAD param: $att";
}
}
# stabilize partial order for consistency and ease-of-debugging:
if (scalar keys %partial) {
- $need = NEED_EML;
- push @op, $OP_EML_NEW if !$seen{$need}++;
+ $need |= NEED_BLOB;
$r[2] = [ map { [ $_, @{$partial{$_}} ] } sort keys %partial ];
}
+ push @op, $OP_EML_NEW if ($need & (EML_HDR|EML_BDY));
+
+ # do we need CRLF conversion?
+ if ($need & CRLF_BREF) {
+ push @op, $OP_CRLF_BREF;
+ } elsif (my $crlf = ($need & (CRLF_HDR|CRLF_BDY))) {
+ if ($crlf == (CRLF_HDR|CRLF_BDY)) {
+ push @op, $OP_CRLF_BREF;
+ } elsif ($need & CRLF_HDR) {
+ push @op, $OP_CRLF_HDR;
+ } else {
+ push @op, $OP_CRLF_BDY;
+ }
+ }
+
$r[0] = $need & NEED_BLOB ? \&fetch_blob :
($need & NEED_SMSG ? \&fetch_smsg : \&fetch_uid);
}
sub msn_to_uid_range ($$) {
- my $uid_base = $_[0]->{uid_base} // 0;
+ my $uid_base = $_[0]->{uid_base};
$_[1] =~ s/([0-9]+)/$uid_base + $1/sge;
}
if (!scalar(keys %$q)) {
$self->msg_more('* SEARCH');
my $beg = 1;
- my $end = $ibx->mm->max // 0;
+ my $end = $ibx->over->max;
uid_clamp($self, \$beg, \$end);
long_response($self, \&uid_search_uid_range,
$tag, \$beg, $end, $sql);
} elsif (my $uid = $q->{uid}) {
if ($uid =~ /\A([0-9]+):([0-9]+|\*)\z/s) {
my ($beg, $end) = ($1, $2);
- $end = $ibx->mm->max if $end eq '*';
+ $end = $ibx->over->max if $end eq '*';
uid_clamp($self, \$beg, \$end);
$self->msg_more('* SEARCH');
long_response($self, \&uid_search_uid_range,