From: Eric Wong <e@yhbt.net>
Date: Wed, 10 Jun 2020 07:04:39 +0000 (+0000)
Subject: imap: start doing iterative config reloading
X-Git-Tag: v1.6.0~425
X-Git-Url: http://www.git.stargrave.org/?p=public-inbox.git;a=commitdiff_plain;h=48180dbb004b5f59b2e80613b6fa2e5e869316f1

imap: start doing iterative config reloading

This will be used to prevent reloading a giant config with
tens/hundreds of thousands of inboxes from blocking the event
loop.
---

diff --git a/lib/PublicInbox/Config.pm b/lib/PublicInbox/Config.pm
index 458f29b2..c18c9c75 100644
--- a/lib/PublicInbox/Config.pm
+++ b/lib/PublicInbox/Config.pm
@@ -99,6 +99,21 @@ sub each_inbox {
 	}
 }
 
+sub iterate_start {
+	my ($self, $cb, $arg) = @_;
+	my $i = 0;
+	$self->{-iter} = [ \$i, $cb, $arg ];
+}
+
+# for PublicInbox::DS::next_tick
+sub event_step {
+	my ($self) = @_;
+	my ($i, $cb, $arg) = @{$self->{-iter}};
+	my $section = $self->{-section_order}->[$$i++];
+	delete($self->{-iter}) unless defined($section);
+	$cb->($self, $section, $arg);
+}
+
 sub lookup_newsgroup {
 	my ($self, $ng) = @_;
 	_lookup_fill($self, '-by_newsgroup', lc($ng));
diff --git a/lib/PublicInbox/IMAPD.pm b/lib/PublicInbox/IMAPD.pm
index 966879d8..a10fabff 100644
--- a/lib/PublicInbox/IMAPD.pm
+++ b/lib/PublicInbox/IMAPD.pm
@@ -7,6 +7,8 @@ package PublicInbox::IMAPD;
 use strict;
 use parent qw(PublicInbox::NNTPD);
 use PublicInbox::InboxIdle;
+use PublicInbox::IMAP;
+# *UID_BLOCK = \&PublicInbox::IMAP::UID_BLOCK;
 
 sub new {
 	my ($class) = @_;
@@ -51,17 +53,64 @@ sub refresh_inboxlist ($) {
 	$self->{inboxlist} = \@names;
 }
 
-sub refresh_groups {
-	my ($self) = @_;
-	my $pi_config = $self->{pi_config} = PublicInbox::Config->new;
-	$self->SUPER::refresh_groups($pi_config);
-	refresh_inboxlist($self);
+sub imapd_refresh_ibx { # pi_config->each_inbox cb
+	my ($ibx, $imapd) = @_;
+	my $ngname = $ibx->{newsgroup} or return;
+	if (ref $ngname) {
+		warn 'multiple newsgroups not supported: '.
+			join(', ', @$ngname). "\n";
+	} elsif ($ngname =~ m![^a-z0-9/_\.\-\~\@\+\=:]! ||
+		 $ngname =~ /\.[0-9]+-[0-9]+\z/) {
+		warn "mailbox name invalid: `$ngname'\n";
+	}
+
+	my $mm = $ibx->mm or return;
+	$ibx->{mm} = undef;
+	defined($ibx->{uidvalidity} = $mm->created_at) or return;
+	$imapd->{tmp_groups}->{$ngname} = $ibx;
 
-	if (my $idler = $self->{idler}) {
+	# preload to avoid fragmentation:
+	$ibx->description;
+	$ibx->base_url;
+	# my $max = $mm->max // 0;
+	# my $uid_min = UID_BLOCK * int($max/UID_BLOCK) + 1;
+}
+
+sub imapd_refresh_finalize {
+	my ($imapd, $pi_config) = @_;
+	$imapd->{groups} = delete $imapd->{tmp_groups};
+	$imapd->{grouplist} = [ values %{$imapd->{groups}} ];
+	refresh_inboxlist($imapd);
+	$imapd->{pi_config} = $pi_config;
+	if (my $idler = $imapd->{idler}) {
 		$idler->refresh($pi_config);
 	}
 }
 
+sub imapd_refresh_step { # pi_config->iterate_start cb
+	my ($pi_config, $section, $imapd) = @_;
+	if (defined($section)) {
+		return if $section !~ m!\Apublicinbox\.([^/]+)\z!;
+		my $ibx = $pi_config->lookup_name($1) or return;
+		imapd_refresh_ibx($ibx, $imapd);
+	} else { # "EOF"
+		imapd_refresh_finalize($imapd, $pi_config);
+	}
+}
+
+sub refresh_groups {
+	my ($self, $sig) = @_;
+	my $pi_config = PublicInbox::Config->new;
+	$self->{tmp_groups} = {};
+	if (0 && $sig) { # SIGHUP
+		$pi_config->iterate_start(\&imapd_refresh_step, $self);
+		PublicInbox::DS::requeue($pi_config); # call event_step
+	} else { # initial start
+		$pi_config->each_inbox(\&imapd_refresh_ibx, $self);
+		imapd_refresh_finalize($self, $pi_config);
+	}
+}
+
 sub idler_start {
 	$_[0]->{idler} //= PublicInbox::InboxIdle->new($_[0]->{pi_config});
 }
diff --git a/script/public-inbox-imapd b/script/public-inbox-imapd
index 63f865f5..60f2e6d8 100644
--- a/script/public-inbox-imapd
+++ b/script/public-inbox-imapd
@@ -9,6 +9,6 @@ use PublicInbox::IMAPdeflate; # loads PublicInbox::IMAP
 use PublicInbox::IMAPD;
 my $imapd = PublicInbox::IMAPD->new;
 PublicInbox::Daemon::run('0.0.0.0:143',
-	sub { $imapd->refresh_groups }, # refresh
+	sub { $imapd->refresh_groups(@_) }, # refresh
 	sub ($$$) { PublicInbox::IMAP->new($_[0], $imapd) }, # post_accept
 	$imapd);