imapd: use nntpd_cache to speed up startup/reload time

ConfigIter was still too slow despite being fair.  The addition of
ART_MIN in ALL->misc means it can be used as a startup/reload cache
for -imapd, too.

This results in a ~3x faster startup for -imapd with 50K inboxes.

diff --git a/lib/PublicInbox/IMAP.pm b/lib/PublicInbox/IMAP.pm
index 19ead70c9d7c3968c3e5d4f244446079c869b488..9955984b0997e28ba86ce196903f2b6e920d3650 100644 (file)
--- a/lib/PublicInbox/IMAP.pm
+++ b/lib/PublicInbox/IMAP.pm
@@ -350,12 +350,12 @@ sub idle_done ($$) {
        "$idle_tag OK Idle done\r\n";
 }
 
-sub ensure_slices_exist ($$$) {
-       my ($imapd, $ibx, $max) = @_;
-       defined(my $mb_top = $ibx->{newsgroup}) or return;
+sub ensure_slices_exist ($$) {
+       my ($imapd, $ibx) = @_;
+       my $mb_top = $ibx->{newsgroup} // return;
        my $mailboxes = $imapd->{mailboxes};
        my @created;
-       for (my $i = int($max/UID_SLICE); $i >= 0; --$i) {
+       for (my $i = int($ibx->art_max/UID_SLICE); $i >= 0; --$i) {
                my $sub_mailbox = "$mb_top.$i";
                last if exists $mailboxes->{$sub_mailbox};
                $mailboxes->{$sub_mailbox} = $ibx;
@@ -387,7 +387,8 @@ sub inbox_lookup ($$;$) {
                        my $uid_end = $uid_base + UID_SLICE;
                        $exists = $over->imap_exists($uid_base, $uid_end);
                }
-               ensure_slices_exist($self->{imapd}, $ibx, $over->max);
+               delete $ibx->{-art_max};
+               ensure_slices_exist($self->{imapd}, $ibx);
        } else {
                if ($examine) {
                        $self->{uid_base} = $uid_base;
@@ -396,9 +397,9 @@ sub inbox_lookup ($$;$) {
                }
                # if "INBOX.foo.bar" is selected and "INBOX.foo.bar.0",
                # check for new UID ranges (e.g. "INBOX.foo.bar.1")
-               if (my $z = $self->{imapd}->{mailboxes}->{"$mailbox.0"}) {
-                       ensure_slices_exist($self->{imapd}, $z,
-                                               $z->over(1)->max);
+               if (my $ibx = $self->{imapd}->{mailboxes}->{"$mailbox.0"}) {
+                       delete $ibx->{-art_max};
+                       ensure_slices_exist($self->{imapd}, $ibx);
                }
        }
        ($ibx, $exists, $uidmax + 1, $uid_base);

diff --git a/lib/PublicInbox/IMAPD.pm b/lib/PublicInbox/IMAPD.pm
index 6038fd88a6e32fd2cd1f7064b61ecfaa0fb39b3d..5368ff040edd457111cdb3984f722ebd67b5904f 100644 (file)
--- a/lib/PublicInbox/IMAPD.pm
+++ b/lib/PublicInbox/IMAPD.pm
@@ -6,7 +6,6 @@ package PublicInbox::IMAPD;
 use strict;
 use v5.10.1;
 use PublicInbox::Config;
-use PublicInbox::ConfigIter;
 use PublicInbox::InboxIdle;
 use PublicInbox::IMAP;
 use PublicInbox::DummyInbox;
@@ -15,7 +14,7 @@ my $dummy = bless { uidvalidity => 0 }, 'PublicInbox::DummyInbox';
 sub new {
        my ($class) = @_;
        bless {
-               mailboxes => {},
+               # mailboxes => {},
                err => \*STDERR,
                out => \*STDOUT,
                # ssl_ctx_opt => { SSL_cert_file => ..., SSL_key_file => ... }
@@ -25,53 +24,45 @@ sub new {
 }
 
 sub imapd_refresh_ibx { # pi_cfg->each_inbox cb
-       my ($ibx, $imapd) = @_;
-       my $ngname = $ibx->{newsgroup} or return;
+       my ($ibx, $imapd, $cache, $dummies) = @_;
+       my $ngname = $ibx->{newsgroup} // return;
 
        # We require lower-case since IMAP mailbox names are
        # case-insensitive (but -nntpd matches INN in being
-       # case-sensitive
+       # case-sensitive)
        if ($ngname =~ m![^a-z0-9/_\.\-\~\@\+\=:]! ||
                        # don't confuse with 50K slices
                        $ngname =~ /\.[0-9]+\z/) {
                warn "mailbox name invalid: newsgroup=`$ngname'\n";
                return;
        }
-       $ibx->over or return;
-       $ibx->{over} = undef;
-
-       # RFC 3501 2.3.1.1 -  "A good UIDVALIDITY value to use in
-       # this case is a 32-bit representation of the creation
-       # date/time of the mailbox"
-       eval { $ibx->uidvalidity };
-       my $mm = delete($ibx->{mm}) or return;
-       defined($ibx->{uidvalidity}) or return;
-       PublicInbox::IMAP::ensure_slices_exist($imapd, $ibx, $mm->max);
-
-       # preload to avoid fragmentation:
-       $ibx->description;
-       $ibx->base_url;
-
-       # ensure dummies are selectable
-       my $dummies = $imapd->{dummies};
-       do {
-               $dummies->{$ngname} = $dummy;
-       } while ($ngname =~ s/\.[^\.]+\z//);
+       my $ce = $cache->{$ngname};
+       %$ibx = (%$ibx, %$ce) if $ce;
+       # only valid if msgmap and over works:
+       if (defined($ibx->uidvalidity)) {
+               # fill ->{mailboxes}:
+               PublicInbox::IMAP::ensure_slices_exist($imapd, $ibx);
+               # preload to avoid fragmentation:
+               $ibx->description;
+               $ibx->base_url;
+               # ensure dummies are selectable:
+               do {
+                       $dummies->{$ngname} = $dummy;
+               } while ($ngname =~ s/\.[^\.]+\z//);
+       }
+       delete @$ibx{qw(mm over)};
 }
 
-sub imapd_refresh_finalize {
-       my ($imapd, $pi_cfg) = @_;
-       my $mailboxes;
-       if (my $next = delete $imapd->{imapd_next}) {
-               $imapd->{mailboxes} = delete $next->{mailboxes};
-               $mailboxes = delete $next->{dummies};
-       } else {
-               $mailboxes = delete $imapd->{dummies};
-       }
-       %$mailboxes = (%$mailboxes, %{$imapd->{mailboxes}});
-       $imapd->{mailboxes} = $mailboxes;
-       $imapd->{mailboxlist} = [
-               map { $_->[2] }
+sub refresh_groups {
+       my ($self, $sig) = @_;
+       my $pi_cfg = PublicInbox::Config->new;
+       my $mailboxes = $self->{mailboxes} = {};
+       my $cache = eval { $pi_cfg->ALL->misc->nntpd_cache_load } // {};
+       my $dummies = {};
+       $pi_cfg->each_inbox(\&imapd_refresh_ibx, $self, $cache, $dummies);
+       %$dummies = (%$dummies, %$mailboxes);
+       $mailboxes = $self->{mailboxes} = $dummies;
+       @{$self->{mailboxlist}} = map { $_->[2] }
                sort { $a->[0] cmp $b->[0] || $a->[1] <=> $b->[1] }
                map {
                        my $u = $_; # capitalize "INBOX" for user-familiarity
@@ -85,40 +76,13 @@ sub imapd_refresh_finalize {
                                [ $1, $2 + 0,
                                  qq[* LIST (\\HasNoChildren) "." $u\r\n] ]
                        }
-               } keys %$mailboxes
-       ];
-       $imapd->{pi_cfg} = $pi_cfg;
-       if (my $idler = $imapd->{idler}) {
+               } keys %$mailboxes;
+       $self->{pi_cfg} = $pi_cfg;
+       if (my $idler = $self->{idler}) {
                $idler->refresh($pi_cfg);
        }
 }
 
-sub imapd_refresh_step { # PublicInbox::ConfigIter cb
-       my ($pi_cfg, $section, $imapd) = @_;
-       if (defined($section)) {
-               return if $section !~ m!\Apublicinbox\.([^/]+)\z!;
-               my $ibx = $pi_cfg->lookup_name($1) or return;
-               imapd_refresh_ibx($ibx, $imapd->{imapd_next});
-       } else { # undef == "EOF"
-               imapd_refresh_finalize($imapd, $pi_cfg);
-       }
-}
-
-sub refresh_groups {
-       my ($self, $sig) = @_;
-       my $pi_cfg = PublicInbox::Config->new;
-       if ($sig) { # SIGHUP is handled through the event loop
-               $self->{imapd_next} = { dummies => {}, mailboxes => {} };
-               my $iter = PublicInbox::ConfigIter->new($pi_cfg,
-                                               \&imapd_refresh_step, $self);
-               $iter->event_step;
-       } else { # initial start is synchronous
-               $self->{dummies} = {};
-               $pi_cfg->each_inbox(\&imapd_refresh_ibx, $self);
-               imapd_refresh_finalize($self, $pi_cfg);
-       }
-}
-
 sub idler_start {
        $_[0]->{idler} //= PublicInbox::InboxIdle->new($_[0]->{pi_cfg});
 }