MANIFEST | 1 + lib/PublicInbox/LEI.pm | 3 +++ lib/PublicInbox/LeiInput.pm | 2 +- lib/PublicInbox/LeiLsMailSource.pm | 82 +++++++++++++++++++++++++++++++++++++++++++++++++++++ lib/PublicInbox/NetReader.pm | 15 ++++++++------- t/lei-import-imap.t | 6 ++++++ t/lei-import-nntp.t | 7 +++++++ diff --git a/MANIFEST b/MANIFEST index 3d4c6cbd91668b9da3916d220484214402ff4a67..d4b3e75df9e0bea3e6ee77a053c15372577c1521 100644 --- a/MANIFEST +++ b/MANIFEST @@ -215,6 +215,7 @@ lib/PublicInbox/LeiInput.pm lib/PublicInbox/LeiInspect.pm lib/PublicInbox/LeiLcat.pm lib/PublicInbox/LeiLsLabel.pm +lib/PublicInbox/LeiLsMailSource.pm lib/PublicInbox/LeiLsMailSync.pm lib/PublicInbox/LeiLsSearch.pm lib/PublicInbox/LeiMailSync.pm diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm index d34997fdc42b98581998029a5a868690a6e95278..833d9c4dde67f2d7c88159c22d6084c8748360db 100644 --- a/lib/PublicInbox/LEI.pm +++ b/lib/PublicInbox/LEI.pm @@ -196,6 +196,8 @@ qw(format|f=s z|0 globoff|g invert-match|v local remote), @c_opt ], 'ls-label' => [ '', 'list labels', qw(z|0 stats:s), @c_opt ], 'ls-mail-sync' => [ '[FILTER]', 'list mail sync folders', qw(z|0 globoff|g invert-match|v local remote), @c_opt ], +'ls-mail-source' => [ 'URL', 'list IMAP or NNTP mail source folders', + qw(z|0 ascii l), @c_opt ], 'forget-external' => [ 'LOCATION...|--prune', 'exclude further results from a publicinbox|extindex', qw(prune), @c_opt ], @@ -381,6 +383,7 @@ 'in-format|F=s' => $stdin_formats, 'format|f=s ls-search' => ['OUT|json|jsonl|concatjson', 'listing output format' ], 'l ls-search' => 'long listing format', +'l ls-mail-source' => 'long listing format', 'format|f=s ls-external' => $ls_format, 'limit|n=i@' => ['NUM', 'limit on number of matches (default: 10000)' ], diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm index 24211bf0cb0dff8a056cff8b84bb85438521ebfc..92d67715194b4f10bdc7750793b3e656a2fdf773 100644 --- a/lib/PublicInbox/LeiInput.pm +++ b/lib/PublicInbox/LeiInput.pm @@ -229,7 +229,7 @@ my $input_path = $input; if ($input =~ m!\A(?:imaps?|nntps?|s?news)://!i) { require PublicInbox::NetReader; $net //= PublicInbox::NetReader->new; - $net->add_url($input); + $net->add_url($input, $self->{-ls_ok}); push @{$sync->{ok}}, $input if $sync; } elsif ($input_path =~ m!\Ahttps?://!i) { # mboxrd.gz # TODO: how would we detect r/w JMAP? diff --git a/lib/PublicInbox/LeiLsMailSource.pm b/lib/PublicInbox/LeiLsMailSource.pm new file mode 100644 index 0000000000000000000000000000000000000000..e95c0b34cb5e4d0c9441e364a2ead66848bcb547 --- /dev/null +++ b/lib/PublicInbox/LeiLsMailSource.pm @@ -0,0 +1,82 @@ +# Copyright (C) 2021 all contributors +# License: AGPL-3.0+ + +# command for listing NNTP groups and IMAP folders, +# handy for users with git-credential-helper configured +# TODO: list JMAP labels +package PublicInbox::LeiLsMailSource; +use strict; +use v5.10.1; +use parent qw(PublicInbox::IPC PublicInbox::LeiInput); + +sub input_path_url { # overrides LeiInput version + my ($self, $url) = @_; + # TODO: support ndjson and other JSONs we support elsewhere + my $json; + my $lei = $self->{lei}; + my $ORS = "\n"; + if ($self->{lei}->{opt}->{l}) { + $json = ref(PublicInbox::Config->json)->new->utf8->canonical; + $json->ascii(1) if $lei->{opt}->{ascii}; + } elsif ($self->{lei}->{opt}->{z}) { + $ORS = "\0"; + } + if ($url =~ m!\Aimaps?://!i) { + my $uri = PublicInbox::URIimap->new($url); + my $mic = $lei->{net}->mic_get($uri); + my $l = $mic->folders_hash($uri->path); # server-side filter + if ($json) { + $lei->puts($json->encode($l)); + } else { + $lei->out(join($ORS, (map { $_->{name} } @$l), '')); + } + } elsif ($url =~ m!\A(?:nntps?|s?news)://!i) { + my $uri = PublicInbox::URInntps->new($url); + my $nn = $lei->{net}->nn_get($uri); + my $l = $nn->newsgroups($uri->group); # name => description + if ($json) { + my $all = $nn->list; + my @x; + for my $ng (sort keys %$l) { + my $desc = $l->{$ng}; + +# we need to drop CR ourselves iff using IO::Socket::SSL since +# Net::Cmd::getline doesn't get used by Net::NNTP if TLS is in play, noted in: +# + $desc =~ s/\r\z//; + + my ($hwm, $lwm, $status) = @{$all->{$ng}}; + push @x, { name => $ng, lwm => $lwm + 0, + hwm => $hwm + 0, status => $status, + description => $desc }; + } + $lei->puts($json->encode(\@x)); + } else { + $lei->out(join($ORS, sort(keys %$l), '')); + } + } else { die "BUG: $url not supported" } +} + +sub lei_ls_mail_source { + my ($lei, $url, $pfx) = @_; + $url =~ m!\A(?:imaps?|nntps?|s?news)://!i or return + $lei->fail('only NNTP and IMAP URLs supported'); + my $self = bless { pfx => $pfx, -ls_ok => 1 }, __PACKAGE__; + $self->prepare_inputs($lei, [ $url ]) or return; + $lei->start_pager if -t $lei->{1}; + my $ops = {}; + $lei->{auth}->op_merge($ops, $self); + my $j = $self->{-wq_nr_workers} = 1; # locked + (my $op_c, $ops) = $lei->workers_start($self, $j, $ops); + $lei->{wq1} = $self; + $lei->{-err_type} = 'non-fatal'; + net_merge_all_done($self) unless $lei->{auth}; + $lei->wait_wq_events($op_c, $ops); # net_merge_all_done if !{auth} +} + +no warnings 'once'; +*ipc_atfork_child = \&PublicInbox::LeiInput::input_only_atfork_child; +*net_merge_all_done = \&PublicInbox::LeiInput::input_only_net_merge_all_done; +*net_merge_all = \&PublicInbox::LeiAuth::net_merge_all; + +1; diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm index 2795a9d4d85e4bbc21c7669a13825db1d6870ba8..bcc6cbf084db58689e97f3ae15d6727367432265 100644 --- a/lib/PublicInbox/NetReader.pm +++ b/lib/PublicInbox/NetReader.pm @@ -236,18 +236,19 @@ $nn; } sub imap_uri { - my ($url) = @_; + my ($url, $ls_ok) = @_; require PublicInbox::URIimap; my $uri = PublicInbox::URIimap->new($url); - $uri ? $uri->canonical : undef; + $uri && ($ls_ok || $uri->mailbox) ? $uri->canonical : undef; } my %IS_NNTP = (news => 1, snews => 1, nntp => 1, nntps => 1); sub nntp_uri { - my ($url) = @_; + my ($url, $ls_ok) = @_; require PublicInbox::URInntps; my $uri = PublicInbox::URInntps->new($url); - $uri && $IS_NNTP{$uri->scheme} && $uri->group ? $uri->canonical : undef; + $uri && $IS_NNTP{$uri->scheme} && ($ls_ok || $uri->group) ? + $uri->canonical : undef; } sub cfg_intvl ($$$) { @@ -367,11 +368,11 @@ \%nn; # for optional {nn_cached} } sub add_url { - my ($self, $arg) = @_; + my ($self, $arg, $ls_ok) = @_; my $uri; - if ($uri = imap_uri($arg)) { + if ($uri = imap_uri($arg, $ls_ok)) { push @{$self->{imap_order}}, $uri; - } elsif ($uri = nntp_uri($arg)) { + } elsif ($uri = nntp_uri($arg, $ls_ok)) { push @{$self->{nntp_order}}, $uri; } else { push @{$self->{unsupported_url}}, $arg; diff --git a/t/lei-import-imap.t b/t/lei-import-imap.t index 34fd6cf999a714afda741ceb14d79f772dc430b7..12f6fad0bd88a075f7dcd0a10ae57cff18436d8a 100644 --- a/t/lei-import-imap.t +++ b/t/lei-import-imap.t @@ -14,6 +14,12 @@ undef $sock; test_lei({ tmpdir => $tmpdir }, sub { my $url = "imap://$host_port/t.v2.0"; my $url_orig = $url; + lei_ok(qw(ls-mail-source), "imap://$host_port/"); + like($lei_out, qr/^t\.v2\.0$/ms, 'shows mailbox'); + lei_ok(qw(ls-mail-source), $url); + is($lei_out, "t.v2.0\n", 'shows only mailbox with filter'); + lei_ok(qw(ls-mail-source -l), "imap://$host_port/"); + is(ref(json_utf8->decode($lei_out)), 'ARRAY', 'ls-mail-source JSON'); lei_ok(qw(q z:1..)); my $out = json_utf8->decode($lei_out); diff --git a/t/lei-import-nntp.t b/t/lei-import-nntp.t index 662da30939fa8e2b88ee64d4b3c04a64c26ad527..f2c3540624290f6e0dc170325448bd2b4a502d7d 100644 --- a/t/lei-import-nntp.t +++ b/t/lei-import-nntp.t @@ -17,6 +17,13 @@ lei_ok(qw(q z:1..)); my $out = json_utf8->decode($lei_out); is_deeply($out, [ undef ], 'nothing imported, yet'); my $url = "nntp://$host_port/t.v2"; + lei_ok(qw(ls-mail-source), "nntp://$host_port/"); + like($lei_out, qr/^t\.v2$/ms, 'shows newsgroup'); + lei_ok(qw(ls-mail-source), $url); + is($lei_out, "t.v2\n", 'shows only newsgroup with filter'); + lei_ok(qw(ls-mail-source -l), "nntp://$host_port/"); + is(ref(json_utf8->decode($lei_out)), 'ARRAY', 'ls-mail-source JSON'); + lei_ok('import', $url); lei_ok(qw(q z:1..)); $out = json_utf8->decode($lei_out);