+ $uri && ($ls_ok || $uri->mailbox) ? $uri->canonical : undef;
+}
+
+my %IS_NNTP = (news => 1, snews => 1, nntp => 1, nntps => 1);
+sub nntp_uri {
+ my ($url, $ls_ok) = @_;
+ require PublicInbox::URInntps;
+ my $uri = PublicInbox::URInntps->new($url);
+ $uri && $IS_NNTP{$uri->scheme} && ($ls_ok || $uri->group) ?
+ $uri->canonical : undef;
+}
+
+sub cfg_intvl ($$$) {
+ my ($cfg, $key, $url) = @_;
+ my $v = $cfg->urlmatch($key, $url) // return;
+ $v =~ /\A[0-9]+(?:\.[0-9]+)?\z/s and return $v + 0;
+ if (ref($v) eq 'ARRAY') {
+ $v = join(', ', @$v);
+ warn "W: $key has multiple values: $v\nW: $key ignored\n";
+ } else {
+ warn "W: $key=$v is not a numeric value in seconds\n";
+ }
+}
+
+sub cfg_bool ($$$) {
+ my ($cfg, $key, $url) = @_;
+ my $orig = $cfg->urlmatch($key, $url) // return;
+ my $bool = $cfg->git_bool($orig);
+ warn "W: $key=$orig for $url is not boolean\n" unless defined($bool);
+ $bool;
+}
+
+# flesh out common IMAP-specific data structures
+sub imap_common_init ($;$) {
+ my ($self, $lei) = @_;
+ return unless $self->{imap_order};
+ $self->{quiet} = 1 if $lei && $lei->{opt}->{quiet};
+ eval { require PublicInbox::IMAPClient } or
+ die "Mail::IMAPClient is required for IMAP:\n$@\n";
+ ($lei || eval { require PublicInbox::IMAPTracker }) or
+ die "DBD::SQLite is required for IMAP\n:$@\n";
+ require PublicInbox::URIimap;
+ my $cfg = $self->{pi_cfg} // $lei->_lei_cfg;
+ my $mic_common = {}; # scheme://authority => Mail:IMAPClient arg
+ for my $uri (@{$self->{imap_order}}) {
+ my $sec = uri_section($uri);
+
+ # knobs directly for Mail::IMAPClient->new
+ for my $k (qw(Starttls Debug Compress Ignoresizeerrors)) {
+ my $bool = cfg_bool($cfg, "imap.$k", $$uri) // next;
+ $mic_common->{$sec}->{$k} = $bool;
+ }
+ my $to = cfg_intvl($cfg, 'imap.timeout', $$uri);
+ $mic_common->{$sec}->{Timeout} = $to if $to;
+
+ # knobs we use ourselves:
+ my $sa = socks_args($cfg->urlmatch('imap.Proxy', $$uri));
+ $self->{cfg_opt}->{$sec}->{-proxy_cfg} = $sa if $sa;
+ for my $k (qw(pollInterval idleInterval)) {
+ $to = cfg_intvl($cfg, "imap.$k", $$uri) // next;
+ $self->{cfg_opt}->{$sec}->{$k} = $to;
+ }
+ my $k = 'imap.fetchBatchSize';
+ my $bs = $cfg->urlmatch($k, $$uri) // next;
+ if ($bs =~ /\A([0-9]+)\z/ && $bs > 0) {
+ $self->{cfg_opt}->{$sec}->{batch_size} = $bs;
+ } else {
+ warn "$k=$bs is not a positive integer\n";
+ }
+ }
+ # make sure we can connect and cache the credentials in memory
+ my $mics = {}; # schema://authority => IMAPClient obj
+ for my $orig_uri (@{$self->{imap_order}}) {
+ my $sec = uri_section($orig_uri);
+ my $uri = PublicInbox::URIimap->new("$sec/");
+ my $mic = $mics->{$sec} //=
+ mic_for($self, $uri, $mic_common, $lei) //
+ die "Unable to continue\n";
+ next unless $self->isa('PublicInbox::NetWriter');
+ my $dst = $orig_uri->mailbox // next;
+ next if $mic->exists($dst); # already exists
+ $mic->create($dst) or die "CREATE $dst failed <$orig_uri>: $@";
+ }
+ $mics;
+}
+
+# flesh out common NNTP-specific data structures
+sub nntp_common_init ($;$) {
+ my ($self, $lei) = @_;
+ return unless $self->{nntp_order};
+ $self->{quiet} = 1 if $lei && $lei->{opt}->{quiet};
+ eval { require Net::NNTP } or
+ die "Net::NNTP is required for NNTP:\n$@\n";
+ ($lei || eval { require PublicInbox::IMAPTracker }) or
+ die "DBD::SQLite is required for NNTP\n:$@\n";
+ my $cfg = $self->{pi_cfg} // $lei->_lei_cfg;
+ my $nn_common = {}; # scheme://authority => Net::NNTP->new arg
+ for my $uri (@{$self->{nntp_order}}) {
+ my $sec = uri_section($uri);
+ my $args = $nn_common->{$sec} //= {};
+
+ # Debug and Timeout are passed to Net::NNTP->new
+ my $v = cfg_bool($cfg, 'nntp.Debug', $$uri);
+ $args->{Debug} = $v if defined $v;
+ my $to = cfg_intvl($cfg, 'nntp.Timeout', $$uri);
+ $args->{Timeout} = $to if $to;
+ my $sa = socks_args($cfg->urlmatch('nntp.Proxy', $$uri));
+ %$args = (%$args, %$sa) if $sa;
+
+ # Net::NNTP post-connect commands
+ for my $k (qw(starttls compress)) {
+ $v = cfg_bool($cfg, "nntp.$k", $$uri) // next;
+ $self->{cfg_opt}->{$sec}->{$k} = $v;
+ }
+
+ # -watch internal option
+ for my $k (qw(pollInterval)) {
+ $to = cfg_intvl($cfg, "nntp.$k", $$uri) // next;
+ $self->{cfg_opt}->{$sec}->{$k} = $to;
+ }
+ }
+ # make sure we can connect and cache the credentials in memory
+ my %nn; # schema://authority => Net::NNTP object
+ for my $uri (@{$self->{nntp_order}}) {
+ my $sec = uri_section($uri);
+ $nn{$sec} //= nn_for($self, $uri, $nn_common, $lei);
+ }
+ \%nn; # for optional {nn_cached}
+}
+
+sub add_url {
+ my ($self, $arg, $ls_ok) = @_;
+ my $uri;
+ if ($uri = imap_uri($arg, $ls_ok)) {
+ $_[1] = $$uri; # canonicalized
+ push @{$self->{imap_order}}, $uri;
+ } elsif ($uri = nntp_uri($arg, $ls_ok)) {
+ $_[1] = $$uri; # canonicalized
+ push @{$self->{nntp_order}}, $uri;
+ } else {
+ push @{$self->{unsupported_url}}, $arg;
+ }
+}
+
+sub errors {
+ my ($self, $lei) = @_;
+ if (my $u = $self->{unsupported_url}) {
+ return "Unsupported URL(s): @$u";
+ }
+ if ($self->{imap_order}) {
+ eval { require PublicInbox::IMAPClient } or
+ die "Mail::IMAPClient is required for IMAP:\n$@\n";
+ }
+ if ($self->{nntp_order}) {
+ eval { require Net::NNTP } or
+ die "Net::NNTP is required for NNTP:\n$@\n";
+ }
+ my $sa = socks_args($lei ? $lei->{opt}->{proxy} : undef);
+ $self->{-proxy_cli} = $sa if $sa;
+ undef;
+}
+
+sub flags2kw ($$$$) {
+ my ($self, $uri, $uid, $flags) = @_;
+ my $kw = [];
+ for my $f (split(/ /, $flags)) {
+ if (my $k = $IMAPflags2kw{$f}) {
+ push @$kw, $k;
+ } elsif ($f eq "\\Recent") { # not in JMAP
+ } elsif ($f eq "\\Deleted") { # not in JMAP
+ return;
+ } elsif ($self->{verbose}) {
+ warn "# unknown IMAP flag $f <$uri/;UID=$uid>\n";
+ }
+ }
+ @$kw = sort @$kw; # for LeiSearch->kw_changed and UI/UX purposes
+ $kw;
+}
+
+sub _imap_do_msg ($$$$$) {
+ my ($self, $uri, $uid, $raw, $flags) = @_;
+ # our target audience expects LF-only, save storage
+ $$raw =~ s/\r\n/\n/sg;
+ my $kw = defined($flags) ?
+ (flags2kw($self, $uri, $uid, $flags) // return) : undef;
+ my ($eml_cb, @args) = @{$self->{eml_each}};
+ $eml_cb->($uri, $uid, $kw, PublicInbox::Eml->new($raw), @args);
+}
+
+sub run_commit_cb ($) {
+ my ($self) = @_;
+ my $cmt_cb_args = $self->{on_commit} or return;
+ my ($cb, @args) = @$cmt_cb_args;
+ $cb->(@args);
+}
+
+sub itrk_last ($$;$$) {
+ my ($self, $uri, $r_uidval, $mic) = @_;
+ return (undef, undef, $r_uidval) unless $self->{incremental};
+ my ($itrk, $l_uid, $l_uidval);
+ if (defined(my $lms = $self->{-lms_ro})) { # LeiMailSync or 0
+ $uri->uidvalidity($r_uidval) if defined $r_uidval;
+ if ($mic) {
+ my $auth = $mic->Authmechanism // '';
+ $uri->auth($auth) if $auth eq 'ANONYMOUS';
+ my $user = $mic->User;
+ $uri->user($user) if defined($user);
+ }
+ my $x;
+ $l_uid = ($lms && ($x = $lms->location_stats($$uri))) ?
+ $x->{'uid.max'} : undef;
+ # itrk remains undef, lei/store worker writes to
+ # mail_sync.sqlite3
+ } else {
+ $itrk = PublicInbox::IMAPTracker->new($$uri);
+ ($l_uidval, $l_uid) = $itrk->get_last($$uri);
+ }
+ ($itrk, $l_uid, $l_uidval //= $r_uidval);
+}
+
+# import flags of already-seen messages
+sub each_old_flags ($$$$) {
+ my ($self, $mic, $uri, $l_uid) = @_;
+ $l_uid ||= 1;
+ my $sec = uri_section($uri);
+ my $bs = ($self->{cfg_opt}->{$sec}->{batch_size} // 1) * 10000;
+ my ($eml_cb, @args) = @{$self->{eml_each}};
+ $self->{quiet} or warn "# $uri syncing flags 1:$l_uid\n";
+ for (my $n = 1; $n <= $l_uid; $n += $bs) {
+ my $end = $n + $bs;
+ $end = $l_uid if $end > $l_uid;
+ my $r = $mic->fetch_hash("$n:$end", 'FLAGS');
+ if (!$r) {
+ return if $!{EINTR} && $self->{quit};
+ return "E: $uri UID FETCH $n:$end error: $!";
+ }
+ while (my ($uid, $per_uid) = each %$r) {
+ my $kw = flags2kw($self, $uri, $uid, $per_uid->{FLAGS})
+ // next;
+ # LeiImport->input_net_cb
+ $eml_cb->($uri, $uid, $kw, undef, @args);
+ }
+ }
+}
+
+# returns true if PERMANENTFLAGS indicates FLAGS of already imported
+# messages are meaningful
+sub perm_fl_ok ($) {
+ my ($perm_fl) = @_;
+ return if !defined($perm_fl);
+ for my $f (split(/[ \t]+/, $perm_fl)) {
+ return 1 if $IMAPflags2kw{$f};
+ }
+ undef;