From e40b7230379f37b31563291bbbee2f0899874fed Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Wed, 3 Aug 2022 20:03:57 +0000 Subject: [PATCH] imapd: use nntpd_cache to speed up startup/reload time ConfigIter was still too slow despite being fair. The addition of ART_MIN in ALL->misc means it can be used as a startup/reload cache for -imapd, too. This results in a ~3x faster startup for -imapd with 50K inboxes. --- lib/PublicInbox/IMAP.pm | 17 +++---- lib/PublicInbox/IMAPD.pm | 100 +++++++++++++-------------------------- 2 files changed, 41 insertions(+), 76 deletions(-) diff --git a/lib/PublicInbox/IMAP.pm b/lib/PublicInbox/IMAP.pm index 19ead70c..9955984b 100644 --- a/lib/PublicInbox/IMAP.pm +++ b/lib/PublicInbox/IMAP.pm @@ -350,12 +350,12 @@ sub idle_done ($$) { "$idle_tag OK Idle done\r\n"; } -sub ensure_slices_exist ($$$) { - my ($imapd, $ibx, $max) = @_; - defined(my $mb_top = $ibx->{newsgroup}) or return; +sub ensure_slices_exist ($$) { + my ($imapd, $ibx) = @_; + my $mb_top = $ibx->{newsgroup} // return; my $mailboxes = $imapd->{mailboxes}; my @created; - for (my $i = int($max/UID_SLICE); $i >= 0; --$i) { + for (my $i = int($ibx->art_max/UID_SLICE); $i >= 0; --$i) { my $sub_mailbox = "$mb_top.$i"; last if exists $mailboxes->{$sub_mailbox}; $mailboxes->{$sub_mailbox} = $ibx; @@ -387,7 +387,8 @@ sub inbox_lookup ($$;$) { my $uid_end = $uid_base + UID_SLICE; $exists = $over->imap_exists($uid_base, $uid_end); } - ensure_slices_exist($self->{imapd}, $ibx, $over->max); + delete $ibx->{-art_max}; + ensure_slices_exist($self->{imapd}, $ibx); } else { if ($examine) { $self->{uid_base} = $uid_base; @@ -396,9 +397,9 @@ sub inbox_lookup ($$;$) { } # if "INBOX.foo.bar" is selected and "INBOX.foo.bar.0", # check for new UID ranges (e.g. "INBOX.foo.bar.1") - if (my $z = $self->{imapd}->{mailboxes}->{"$mailbox.0"}) { - ensure_slices_exist($self->{imapd}, $z, - $z->over(1)->max); + if (my $ibx = $self->{imapd}->{mailboxes}->{"$mailbox.0"}) { + delete $ibx->{-art_max}; + ensure_slices_exist($self->{imapd}, $ibx); } } ($ibx, $exists, $uidmax + 1, $uid_base); diff --git a/lib/PublicInbox/IMAPD.pm b/lib/PublicInbox/IMAPD.pm index 6038fd88..5368ff04 100644 --- a/lib/PublicInbox/IMAPD.pm +++ b/lib/PublicInbox/IMAPD.pm @@ -6,7 +6,6 @@ package PublicInbox::IMAPD; use strict; use v5.10.1; use PublicInbox::Config; -use PublicInbox::ConfigIter; use PublicInbox::InboxIdle; use PublicInbox::IMAP; use PublicInbox::DummyInbox; @@ -15,7 +14,7 @@ my $dummy = bless { uidvalidity => 0 }, 'PublicInbox::DummyInbox'; sub new { my ($class) = @_; bless { - mailboxes => {}, + # mailboxes => {}, err => \*STDERR, out => \*STDOUT, # ssl_ctx_opt => { SSL_cert_file => ..., SSL_key_file => ... } @@ -25,53 +24,45 @@ sub new { } sub imapd_refresh_ibx { # pi_cfg->each_inbox cb - my ($ibx, $imapd) = @_; - my $ngname = $ibx->{newsgroup} or return; + my ($ibx, $imapd, $cache, $dummies) = @_; + my $ngname = $ibx->{newsgroup} // return; # We require lower-case since IMAP mailbox names are # case-insensitive (but -nntpd matches INN in being - # case-sensitive + # case-sensitive) if ($ngname =~ m![^a-z0-9/_\.\-\~\@\+\=:]! || # don't confuse with 50K slices $ngname =~ /\.[0-9]+\z/) { warn "mailbox name invalid: newsgroup=`$ngname'\n"; return; } - $ibx->over or return; - $ibx->{over} = undef; - - # RFC 3501 2.3.1.1 - "A good UIDVALIDITY value to use in - # this case is a 32-bit representation of the creation - # date/time of the mailbox" - eval { $ibx->uidvalidity }; - my $mm = delete($ibx->{mm}) or return; - defined($ibx->{uidvalidity}) or return; - PublicInbox::IMAP::ensure_slices_exist($imapd, $ibx, $mm->max); - - # preload to avoid fragmentation: - $ibx->description; - $ibx->base_url; - - # ensure dummies are selectable - my $dummies = $imapd->{dummies}; - do { - $dummies->{$ngname} = $dummy; - } while ($ngname =~ s/\.[^\.]+\z//); + my $ce = $cache->{$ngname}; + %$ibx = (%$ibx, %$ce) if $ce; + # only valid if msgmap and over works: + if (defined($ibx->uidvalidity)) { + # fill ->{mailboxes}: + PublicInbox::IMAP::ensure_slices_exist($imapd, $ibx); + # preload to avoid fragmentation: + $ibx->description; + $ibx->base_url; + # ensure dummies are selectable: + do { + $dummies->{$ngname} = $dummy; + } while ($ngname =~ s/\.[^\.]+\z//); + } + delete @$ibx{qw(mm over)}; } -sub imapd_refresh_finalize { - my ($imapd, $pi_cfg) = @_; - my $mailboxes; - if (my $next = delete $imapd->{imapd_next}) { - $imapd->{mailboxes} = delete $next->{mailboxes}; - $mailboxes = delete $next->{dummies}; - } else { - $mailboxes = delete $imapd->{dummies}; - } - %$mailboxes = (%$mailboxes, %{$imapd->{mailboxes}}); - $imapd->{mailboxes} = $mailboxes; - $imapd->{mailboxlist} = [ - map { $_->[2] } +sub refresh_groups { + my ($self, $sig) = @_; + my $pi_cfg = PublicInbox::Config->new; + my $mailboxes = $self->{mailboxes} = {}; + my $cache = eval { $pi_cfg->ALL->misc->nntpd_cache_load } // {}; + my $dummies = {}; + $pi_cfg->each_inbox(\&imapd_refresh_ibx, $self, $cache, $dummies); + %$dummies = (%$dummies, %$mailboxes); + $mailboxes = $self->{mailboxes} = $dummies; + @{$self->{mailboxlist}} = map { $_->[2] } sort { $a->[0] cmp $b->[0] || $a->[1] <=> $b->[1] } map { my $u = $_; # capitalize "INBOX" for user-familiarity @@ -85,40 +76,13 @@ sub imapd_refresh_finalize { [ $1, $2 + 0, qq[* LIST (\\HasNoChildren) "." $u\r\n] ] } - } keys %$mailboxes - ]; - $imapd->{pi_cfg} = $pi_cfg; - if (my $idler = $imapd->{idler}) { + } keys %$mailboxes; + $self->{pi_cfg} = $pi_cfg; + if (my $idler = $self->{idler}) { $idler->refresh($pi_cfg); } } -sub imapd_refresh_step { # PublicInbox::ConfigIter cb - my ($pi_cfg, $section, $imapd) = @_; - if (defined($section)) { - return if $section !~ m!\Apublicinbox\.([^/]+)\z!; - my $ibx = $pi_cfg->lookup_name($1) or return; - imapd_refresh_ibx($ibx, $imapd->{imapd_next}); - } else { # undef == "EOF" - imapd_refresh_finalize($imapd, $pi_cfg); - } -} - -sub refresh_groups { - my ($self, $sig) = @_; - my $pi_cfg = PublicInbox::Config->new; - if ($sig) { # SIGHUP is handled through the event loop - $self->{imapd_next} = { dummies => {}, mailboxes => {} }; - my $iter = PublicInbox::ConfigIter->new($pi_cfg, - \&imapd_refresh_step, $self); - $iter->event_step; - } else { # initial start is synchronous - $self->{dummies} = {}; - $pi_cfg->each_inbox(\&imapd_refresh_ibx, $self); - imapd_refresh_finalize($self, $pi_cfg); - } -} - sub idler_start { $_[0]->{idler} //= PublicInbox::InboxIdle->new($_[0]->{pi_cfg}); } -- 2.44.0