X-Git-Url: http://www.git.stargrave.org/?p=public-inbox.git;a=blobdiff_plain;f=lib%2FPublicInbox%2FIMAP.pm;h=805f1102536d99d9cf77807f86e1b0b3e6290ce4;hp=2af5ab0c530fc28b382db4d5f2dbaded26d245e9;hb=23af251dd607c4e75ab1e68063f2c885c48cc035;hpb=5f875446975b1473c1ffd7196e572e13d58ba56f diff --git a/lib/PublicInbox/IMAP.pm b/lib/PublicInbox/IMAP.pm index 2af5ab0c..805f1102 100644 --- a/lib/PublicInbox/IMAP.pm +++ b/lib/PublicInbox/IMAP.pm @@ -1,4 +1,4 @@ -# Copyright (C) 2020 all contributors +# Copyright (C) all contributors # License: AGPL-3.0+ # # Each instance of this represents an IMAP client connected to @@ -36,7 +36,6 @@ use parent qw(PublicInbox::DS); use PublicInbox::Eml; use PublicInbox::EmlContentFoo qw(parse_content_disposition); use PublicInbox::DS qw(now); -use PublicInbox::Syscall qw(EPOLLIN EPOLLONESHOT); use PublicInbox::GitAsyncCat; use Text::ParseWords qw(parse_line); use Errno qw(EAGAIN); @@ -99,33 +98,15 @@ undef %FETCH_NEED; my $valid_range = '[0-9]+|[0-9]+:[0-9]+|[0-9]+:\*'; $valid_range = qr/\A(?:$valid_range)(?:,(?:$valid_range))*\z/; -# RFC 3501 5.4. Autologout Timer needs to be >= 30min -$PublicInbox::DS::EXPTIME = 60 * 30; - -sub greet ($) { +sub do_greet { my ($self) = @_; my $capa = capa($self); $self->write(\"* OK [$capa] public-inbox-imapd ready\r\n"); } -sub new ($$$) { - my ($class, $sock, $imapd) = @_; - my $self = bless { imapd => $imapd }, 'PublicInbox::IMAP_preauth'; - my $ev = EPOLLIN; - my $wbuf; - if ($sock->can('accept_SSL') && !$sock->accept_SSL) { - return CORE::close($sock) if $! != EAGAIN; - $ev = PublicInbox::TLS::epollbit() or return CORE::close($sock); - $wbuf = [ \&PublicInbox::DS::accept_tls_step, \&greet ]; - } - $self->SUPER::new($sock, $ev | EPOLLONESHOT); - if ($wbuf) { - $self->{wbuf} = $wbuf; - } else { - greet($self); - } - $self->update_idle_time; - $self; +sub new { + my (undef, $sock, $imapd) = @_; + (bless { imapd => $imapd }, 'PublicInbox::IMAP_preauth')->greet($sock) } sub logged_in { 1 } @@ -195,14 +176,14 @@ sub cmd_capability ($$) { # but uo2m_hibernate can compact and deduplicate it sub uo2m_ary_new ($;$) { my ($self, $exists) = @_; - my $base = $self->{uid_base}; - my $uids = $self->{ibx}->over->uid_range($base + 1, $base + UID_SLICE); + my $ub = $self->{uid_base}; + my $uids = $self->{ibx}->over(1)->uid_range($ub + 1, $ub + UID_SLICE); # convert UIDs to offsets from {base} my @tmp; # [$UID_OFFSET] => $MSN my $msn = 0; - ++$base; - $tmp[$_ - $base] = ++$msn for @$uids; + ++$ub; + $tmp[$_ - $ub] = ++$msn for @$uids; $$exists = $msn if $exists; \@tmp; } @@ -243,7 +224,7 @@ sub uo2m_extend ($$;$) { # need to extend the current range: my $base = $self->{uid_base}; ++$beg; - my $uids = $self->{ibx}->over->uid_range($beg, $base + UID_SLICE); + my $uids = $self->{ibx}->over(1)->uid_range($beg, $base + UID_SLICE); return $uo2m if !scalar(@$uids); my @tmp; # [$UID_OFFSET] => $MSN my $write_method = $_[2] // 'msg_more'; @@ -316,20 +297,18 @@ sub on_inbox_unlock { } } -# called every X minute(s) or so by PublicInbox::DS::later -my $IDLERS = {}; -my $idle_timer; +# called every minute or so by PublicInbox::DS::later +my $IDLERS; # fileno($obj->{sock}) => PublicInbox::IMAP sub idle_tick_all { my $old = $IDLERS; - $IDLERS = {}; + $IDLERS = undef; for my $i (values %$old) { next if ($i->{wbuf} || !exists($i->{-idle_tag})); - $i->update_idle_time or next; $IDLERS->{fileno($i->{sock})} = $i; $i->write(\"* OK Still here\r\n"); } - $idle_timer = scalar keys %$IDLERS ? - PublicInbox::DS::later(\&idle_tick_all) : undef; + $IDLERS and + PublicInbox::DS::add_uniq_timer('idle', 60, \&idle_tick_all); } sub cmd_idle ($$) { @@ -342,11 +321,11 @@ sub cmd_idle ($$) { my $fd = fileno($sock); $self->{-idle_tag} = $tag; # only do inotify on most recent slice - if ($ibx->over->max < $uid_end) { + if ($ibx->over(1)->max < $uid_end) { $ibx->subscribe_unlock($fd, $self); $self->{imapd}->idler_start; } - $idle_timer //= PublicInbox::DS::later(\&idle_tick_all); + PublicInbox::DS::add_uniq_timer('idle', 60, \&idle_tick_all); $IDLERS->{$fd} = $self; \"+ idling\r\n" } @@ -384,7 +363,7 @@ sub ensure_slices_exist ($$$) { push @created, $sub_mailbox; } return unless @created; - my $l = $imapd->{inboxlist} or return; + my $l = $imapd->{mailboxlist} or return; push @$l, map { qq[* LIST (\\HasNoChildren) "." $_\r\n] } @created; } @@ -393,20 +372,20 @@ sub inbox_lookup ($$;$) { my ($ibx, $exists, $uidmax, $uid_base) = (undef, 0, 0, 0); $mailbox = lc $mailbox; $ibx = $self->{imapd}->{mailboxes}->{$mailbox} or return; - my $over = $ibx->over; + my $over = $ibx->over(1); if ($over != $ibx) { # not a dummy $mailbox =~ /\.([0-9]+)\z/ or die "BUG: unexpected dummy mailbox: $mailbox\n"; $uid_base = $1 * UID_SLICE; - # ->num_highwater caches for writers, so use ->meta_accessor - $uidmax = $ibx->mm->meta_accessor('num_highwater') // 0; + $uidmax = $ibx->mm->num_highwater // 0; if ($examine) { $self->{uid_base} = $uid_base; $self->{ibx} = $ibx; $self->{uo2m} = uo2m_ary_new($self, \$exists); } else { - $exists = $over->imap_exists; + my $uid_end = $uid_base + UID_SLICE; + $exists = $over->imap_exists($uid_base, $uid_end); } ensure_slices_exist($self->{imapd}, $ibx, $over->max); } else { @@ -418,7 +397,8 @@ sub inbox_lookup ($$;$) { # if "INBOX.foo.bar" is selected and "INBOX.foo.bar.0", # check for new UID ranges (e.g. "INBOX.foo.bar.1") if (my $z = $self->{imapd}->{mailboxes}->{"$mailbox.0"}) { - ensure_slices_exist($self->{imapd}, $z, $z->over->max); + ensure_slices_exist($self->{imapd}, $z, + $z->over(1)->max); } } ($ibx, $exists, $uidmax + 1, $uid_base); @@ -499,7 +479,7 @@ sub body_disposition ($) { my $cd = $eml->header_raw('Content-Disposition') or return 'NIL'; $cd = parse_content_disposition($cd); my $buf = '('._esc($cd->{type}); - $buf .= ' ' . _esc_hash(delete $cd->{attributes}); + $buf .= ' ' . _esc_hash($cd->{attributes}); $buf .= ')'; } @@ -511,7 +491,7 @@ sub body_leaf ($$;$) { my $ct = $eml->ct; $buf .= '('._esc($ct->{type}).' '; $buf .= _esc($ct->{subtype}); - $buf .= ' ' . _esc_hash(delete $ct->{attributes}); + $buf .= ' ' . _esc_hash($ct->{attributes}); $buf .= ' ' . _esc($eml->header_raw('Content-ID')); $buf .= ' ' . _esc($eml->header_raw('Content-Description')); my $cte = $eml->header_raw('Content-Transfer-Encoding') // '7bit'; @@ -540,7 +520,7 @@ sub body_parent ($$$) { $buf .= @$hold ? join('', @$hold) : 'NIL'; $buf .= ' '._esc($ct->{subtype}); if ($structure) { - $buf .= ' '._esc_hash(delete $ct->{attributes}); + $buf .= ' '._esc_hash($ct->{attributes}); $buf .= ' '.body_disposition($eml); $buf .= ' '._esc($eml->header_raw('Content-Language')); $buf .= ' '._esc($eml->header_raw('Content-Location')); @@ -583,22 +563,6 @@ sub fetch_body ($;$) { join('', @hold); } -sub requeue_once ($) { - my ($self) = @_; - # COMPRESS users all share the same DEFLATE context. - # Flush it here to ensure clients don't see - # each other's data - $self->zflush; - - # no recursion, schedule another call ASAP, - # but only after all pending writes are done. - # autovivify wbuf: - my $new_size = push(@{$self->{wbuf}}, \&long_step); - - # wbuf may be populated by $cb, no need to rearm if so: - $self->requeue if $new_size == 1; -} - sub fetch_run_ops { my ($self, $smsg, $bref, $ops, $partial) = @_; my $uid = $smsg->{num}; @@ -612,7 +576,7 @@ sub fetch_run_ops { $self->msg_more(")\r\n"); } -sub fetch_blob_cb { # called by git->cat_async via git_async_cat +sub fetch_blob_cb { # called by git->cat_async via ibx_async_cat my ($bref, $oid, $type, $size, $fetch_arg) = @_; my ($self, undef, $msgs, $range_info, $ops, $partial) = @$fetch_arg; my $ibx = $self->{ibx} or return $self->close; # client disconnected @@ -621,17 +585,17 @@ sub fetch_blob_cb { # called by git->cat_async via git_async_cat # it's possible to have TOCTOU if an admin runs # public-inbox-(edit|purge), just move onto the next message warn "E: $smsg->{blob} missing in $ibx->{inboxdir}\n"; - return requeue_once($self); + return $self->requeue_once; } else { $smsg->{blob} eq $oid or die "BUG: $smsg->{blob} != $oid"; } my $pre; if (!$self->{wbuf} && (my $nxt = $msgs->[0])) { - $pre = git_async_prefetch($ibx->git, $nxt->{blob}, - \&fetch_blob_cb, $fetch_arg); + $pre = ibx_async_prefetch($ibx, $nxt->{blob}, + \&fetch_blob_cb, $fetch_arg); } fetch_run_ops($self, $smsg, $bref, $ops, $partial); - $pre ? $self->zflush : requeue_once($self); + $pre ? $self->zflush : $self->requeue_once; } sub emit_rfc822 { @@ -722,7 +686,7 @@ sub range_step ($$) { uid_clamp($self, \$beg, \$end); } elsif ($range =~ /\A([0-9]+):\*\z/) { $beg = $1 + 0; - $end = $self->{ibx}->over->max; + $end = $self->{ibx}->over(1)->max; $end = $uid_end if $end > $uid_end; $beg = $end if $beg > $end; uid_clamp($self, \$beg, \$end); @@ -740,7 +704,7 @@ sub range_step ($$) { sub refill_range ($$$) { my ($self, $msgs, $range_info) = @_; my ($beg, $end, $range_csv) = @$range_info; - if (scalar(@$msgs = @{$self->{ibx}->over->query_xover($beg, $end)})) { + if (scalar(@$msgs = @{$self->{ibx}->over(1)->query_xover($beg, $end)})){ $range_info->[0] = $msgs->[-1]->{num} + 1; return; } @@ -760,7 +724,7 @@ sub fetch_blob { # long_response } } uo2m_extend($self, $msgs->[-1]->{num}); - git_async_cat($self->{ibx}->git, $msgs->[0]->{blob}, + ibx_async_cat($self->{ibx}, $msgs->[0]->{blob}, \&fetch_blob_cb, \@_); } @@ -781,7 +745,7 @@ sub fetch_smsg { # long_response sub refill_uids ($$$;$) { my ($self, $uids, $range_info, $sql) = @_; my ($beg, $end, $range_csv) = @$range_info; - my $over = $self->{ibx}->over; + my $over = $self->{ibx}->over(1); while (1) { if (scalar(@$uids = @{$over->uid_range($beg, $end, $sql)})) { $range_info->[0] = $uids->[-1] + 1; # update $beg @@ -849,7 +813,7 @@ sub cmd_status ($$$;@) { my %patmap = ('*' => '.*', '%' => '[^\.]*'); sub cmd_list ($$$$) { my ($self, $tag, $refname, $wildcard) = @_; - my $l = $self->{imapd}->{inboxlist}; + my $l = $self->{imapd}->{mailboxlist}; if ($refname eq '' && $wildcard eq '') { # request for hierarchy delimiter $l = [ qq[* LIST (\\Noselect) "." ""\r\n] ]; @@ -877,12 +841,12 @@ sub eml_index_offs_i { # PublicInbox::Eml::each_part callback # prepares an index for BODY[$SECTION_IDX] fetches sub eml_body_idx ($$) { my ($eml, $section_idx) = @_; - my $idx = $eml->{imap_all_parts} //= do { + my $idx = $eml->{imap_all_parts} // do { my $all = {}; $eml->each_part(\&eml_index_offs_i, $all, 0, 1); # top-level of multipart, BODY[0] not allowed (nz-number) delete $all->{0}; - $all; + $eml->{imap_all_parts} = $all; }; $idx->{$section_idx}; } @@ -1071,7 +1035,7 @@ sub cmd_uid_fetch ($$$$;@) { my $range_info = range_step($self, \$range_csv); return "$tag $range_info\r\n" if !ref($range_info); uo2m_hibernate($self) if $cb == \&fetch_blob; # slow, save RAM - long_response($self, $cb, $tag, [], $range_info, $ops, $partial); + $self->long_response($cb, $tag, [], $range_info, $ops, $partial); } sub cmd_fetch ($$$$;@) { @@ -1086,7 +1050,7 @@ sub cmd_fetch ($$$$;@) { my $range_info = range_step($self, \$range_csv); return "$tag $range_info\r\n" if !ref($range_info); uo2m_hibernate($self) if $cb == \&fetch_blob; # slow, save RAM - long_response($self, $cb, $tag, [], $range_info, $ops, $partial); + $self->long_response($cb, $tag, [], $range_info, $ops, $partial); } sub msn_convert ($$) { @@ -1114,7 +1078,7 @@ sub parse_imap_query ($$) { my ($self, $query) = @_; my $q = PublicInbox::IMAPsearchqp::parse($self, $query); if (ref($q)) { - my $max = $self->{ibx}->over->max; + my $max = $self->{ibx}->over(1)->max; my $beg = 1; uid_clamp($self, \$beg, \$max); $q->{range_info} = [ $beg, $max ]; @@ -1130,7 +1094,7 @@ sub search_common { my ($sql, $range_info) = delete @$q{qw(sql range_info)}; if (!scalar(keys %$q)) { # overview.sqlite3 $self->msg_more('* SEARCH'); - long_response($self, \&search_uid_range, + $self->long_response(\&search_uid_range, $tag, $sql, $range_info, $want_msn); } elsif ($q = $q->{xap}) { my $srch = $self->{ibx}->isrch or @@ -1142,7 +1106,7 @@ sub search_common { }; my $mset = $srch->mset($q, $opt); my $uids = $srch->mset_to_artnums($mset, $opt); - msn_convert($self, $uids) if $want_msn; + msn_convert($self, $uids) if scalar(@$uids) && $want_msn; "* SEARCH @$uids\r\n$tag OK Search done\r\n"; } else { "$tag BAD Error\r\n"; @@ -1159,15 +1123,6 @@ sub cmd_search ($$$;) { search_common($self, $tag, $query, 1); } -sub args_ok ($$) { # duplicated from PublicInbox::NNTP - my ($cb, $argc) = @_; - my $tot = prototype $cb; - my ($nreq, undef) = split(';', $tot); - $nreq = ($nreq =~ tr/$//) - 1; - $tot = ($tot =~ tr/$//) - 1; - ($argc <= $tot && $argc >= $nreq); -} - # returns 1 if we can continue, 0 if not due to buffered writes or disconnect sub process_line ($$) { my ($self, $l) = @_; @@ -1210,37 +1165,6 @@ sub process_line ($$) { $self->write($res); } -sub long_step { - my ($self) = @_; - # wbuf is unset or empty, here; {long} may add to it - my ($fd, $cb, $t0, @args) = @{$self->{long_cb}}; - my $more = eval { $cb->($self, @args) }; - if ($@ || !$self->{sock}) { # something bad happened... - delete $self->{long_cb}; - my $elapsed = now() - $t0; - if ($@) { - err($self, - "%s during long response[$fd] - %0.6f", - $@, $elapsed); - } - out($self, " deferred[$fd] aborted - %0.6f", $elapsed); - $self->close; - } elsif ($more) { # $self->{wbuf}: - $self->update_idle_time; - - # control passed to git_async_cat if $more == \undef - requeue_once($self) if !ref($more); - } else { # all done! - delete $self->{long_cb}; - my $elapsed = now() - $t0; - my $fd = fileno($self->{sock}); - out($self, " deferred[$fd] done - %0.6f", $elapsed); - my $wbuf = $self->{wbuf}; # do NOT autovivify - - $self->requeue unless $wbuf && @$wbuf; - } -} - sub err ($$;@) { my ($self, $fmt, @args) = @_; printf { $self->{imapd}->{err} } $fmt."\n", @args; @@ -1251,25 +1175,12 @@ sub out ($$;@) { printf { $self->{imapd}->{out} } $fmt."\n", @args; } -sub long_response ($$;@) { - my ($self, $cb, @args) = @_; # cb returns true if more, false if done - - my $sock = $self->{sock} or return; - # make sure we disable reading during a long response, - # clients should not be sending us stuff and making us do more - # work while we are stream a response to them - $self->{long_cb} = [ fileno($sock), $cb, now(), @args ]; - long_step($self); # kick off! - undef; -} - # callback used by PublicInbox::DS for any (e)poll (in/out/hup/err) sub event_step { my ($self) = @_; return unless $self->flush_write && $self->{sock} && !$self->{long_cb}; - $self->update_idle_time; # only read more requests if we've drained the write buffer, # otherwise we can be buffering infinitely w/o backpressure @@ -1295,17 +1206,12 @@ sub event_step { return $self->close if $r < 0; $self->rbuf_idle($rbuf); - $self->update_idle_time; # maybe there's more pipelined data, or we'll have # to register it for socket-readiness notifications $self->requeue unless $pending; } -sub compressed { undef } - -sub zflush {} # overridden by IMAPdeflate - # RFC 4978 sub cmd_compress ($$$) { my ($self, $tag, $alg) = @_; @@ -1315,7 +1221,9 @@ sub cmd_compress ($$$) { # CRIME made TLS compression obsolete # return "$tag NO [COMPRESSIONACTIVE]\r\n" if $self->tls_compressed; - PublicInbox::IMAPdeflate->enable($self, $tag); + PublicInbox::IMAPdeflate->enable($self) or return + \"$tag BAD failed to activate compression\r\n"; + PublicInbox::DS::write($self, \"$tag OK DEFLATE active\r\n"); $self->requeue; undef } @@ -1334,14 +1242,14 @@ sub cmd_starttls ($$) { undef; } -# for graceful shutdown in PublicInbox::Daemon: -sub busy { - my ($self, $now) = @_; +sub busy { # for graceful shutdown in PublicInbox::Daemon: + my ($self) = @_; if (defined($self->{-idle_tag})) { $self->write(\"* BYE server shutting down\r\n"); return; # not busy anymore } - ($self->{rbuf} || $self->{wbuf} || $self->not_idle_long($now)); + defined($self->{rbuf}) || defined($self->{wbuf}) || + !$self->write(\"* BYE server shutting down\r\n"); } sub close { @@ -1361,4 +1269,8 @@ our @ISA = qw(PublicInbox::IMAP); sub logged_in { 0 } +package PublicInbox::IMAPdeflate; +use PublicInbox::DSdeflate; +our @ISA = qw(PublicInbox::DSdeflate PublicInbox::IMAP); + 1;