imap: start doing iterative config reloading

This will be used to prevent reloading a giant config with
tens/hundreds of thousands of inboxes from blocking the event
loop.

diff --git a/lib/PublicInbox/Config.pm b/lib/PublicInbox/Config.pm
index 458f29b2964d32c592f0e33d9c060216efb6aee7..c18c9c75b4ffee3a1fb865ca04d8ce6eda01d330 100644 (file)
--- a/lib/PublicInbox/Config.pm
+++ b/lib/PublicInbox/Config.pm
@@ -99,6 +99,21 @@ sub each_inbox {
        }
 }
 
+sub iterate_start {
+       my ($self, $cb, $arg) = @_;
+       my $i = 0;
+       $self->{-iter} = [ \$i, $cb, $arg ];
+}
+
+# for PublicInbox::DS::next_tick
+sub event_step {
+       my ($self) = @_;
+       my ($i, $cb, $arg) = @{$self->{-iter}};
+       my $section = $self->{-section_order}->[$$i++];
+       delete($self->{-iter}) unless defined($section);
+       $cb->($self, $section, $arg);
+}
+
 sub lookup_newsgroup {
        my ($self, $ng) = @_;
        _lookup_fill($self, '-by_newsgroup', lc($ng));

diff --git a/lib/PublicInbox/IMAPD.pm b/lib/PublicInbox/IMAPD.pm
index 966879d8ea59f76e8401468e09db7953bbc269c2..a10fabffb1fbfa0cce2007635d93badd07b91c4c 100644 (file)
--- a/lib/PublicInbox/IMAPD.pm
+++ b/lib/PublicInbox/IMAPD.pm
@@ -7,6 +7,8 @@ package PublicInbox::IMAPD;
 use strict;
 use parent qw(PublicInbox::NNTPD);
 use PublicInbox::InboxIdle;
+use PublicInbox::IMAP;
+# *UID_BLOCK = \&PublicInbox::IMAP::UID_BLOCK;
 
 sub new {
        my ($class) = @_;
@@ -51,17 +53,64 @@ sub refresh_inboxlist ($) {
        $self->{inboxlist} = \@names;
 }
 
-sub refresh_groups {
-       my ($self) = @_;
-       my $pi_config = $self->{pi_config} = PublicInbox::Config->new;
-       $self->SUPER::refresh_groups($pi_config);
-       refresh_inboxlist($self);
+sub imapd_refresh_ibx { # pi_config->each_inbox cb
+       my ($ibx, $imapd) = @_;
+       my $ngname = $ibx->{newsgroup} or return;
+       if (ref $ngname) {
+               warn 'multiple newsgroups not supported: '.
+                       join(', ', @$ngname). "\n";
+       } elsif ($ngname =~ m![^a-z0-9/_\.\-\~\@\+\=:]! ||
+                $ngname =~ /\.[0-9]+-[0-9]+\z/) {
+               warn "mailbox name invalid: `$ngname'\n";
+       }
+
+       my $mm = $ibx->mm or return;
+       $ibx->{mm} = undef;
+       defined($ibx->{uidvalidity} = $mm->created_at) or return;
+       $imapd->{tmp_groups}->{$ngname} = $ibx;
 
-       if (my $idler = $self->{idler}) {
+       # preload to avoid fragmentation:
+       $ibx->description;
+       $ibx->base_url;
+       # my $max = $mm->max // 0;
+       # my $uid_min = UID_BLOCK * int($max/UID_BLOCK) + 1;
+}
+
+sub imapd_refresh_finalize {
+       my ($imapd, $pi_config) = @_;
+       $imapd->{groups} = delete $imapd->{tmp_groups};
+       $imapd->{grouplist} = [ values %{$imapd->{groups}} ];
+       refresh_inboxlist($imapd);
+       $imapd->{pi_config} = $pi_config;
+       if (my $idler = $imapd->{idler}) {
                $idler->refresh($pi_config);
        }
 }
 
+sub imapd_refresh_step { # pi_config->iterate_start cb
+       my ($pi_config, $section, $imapd) = @_;
+       if (defined($section)) {
+               return if $section !~ m!\Apublicinbox\.([^/]+)\z!;
+               my $ibx = $pi_config->lookup_name($1) or return;
+               imapd_refresh_ibx($ibx, $imapd);
+       } else { # "EOF"
+               imapd_refresh_finalize($imapd, $pi_config);
+       }
+}
+
+sub refresh_groups {
+       my ($self, $sig) = @_;
+       my $pi_config = PublicInbox::Config->new;
+       $self->{tmp_groups} = {};
+       if (0 && $sig) { # SIGHUP
+               $pi_config->iterate_start(\&imapd_refresh_step, $self);
+               PublicInbox::DS::requeue($pi_config); # call event_step
+       } else { # initial start
+               $pi_config->each_inbox(\&imapd_refresh_ibx, $self);
+               imapd_refresh_finalize($self, $pi_config);
+       }
+}
+
 sub idler_start {
        $_[0]->{idler} //= PublicInbox::InboxIdle->new($_[0]->{pi_config});
 }

diff --git a/script/public-inbox-imapd b/script/public-inbox-imapd
index 63f865f53f01f5afc21cae418bf93c476148fffe..60f2e6d8a3698a7f4022000d86ceff3189deed13 100644 (file)
--- a/script/public-inbox-imapd
+++ b/script/public-inbox-imapd
@@ -9,6 +9,6 @@ use PublicInbox::IMAPdeflate; # loads PublicInbox::IMAP
 use PublicInbox::IMAPD;
 my $imapd = PublicInbox::IMAPD->new;
 PublicInbox::Daemon::run('0.0.0.0:143',
-       sub { $imapd->refresh_groups }, # refresh
+       sub { $imapd->refresh_groups(@_) }, # refresh
        sub ($$$) { PublicInbox::IMAP->new($_[0], $imapd) }, # post_accept
        $imapd);